diff --git a/dtests/src/resources/scripts/alterTable/alterTable.sql b/dtests/src/resources/scripts/alterTable/alterTable.sql index 2194568103..8cccb46e2f 100644 --- a/dtests/src/resources/scripts/alterTable/alterTable.sql +++ b/dtests/src/resources/scripts/alterTable/alterTable.sql @@ -4,9 +4,9 @@ describe orders; SELECT count(*) FROM orders; -insert into app.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuailles en stock 2','rue du Commerce','Lyon',NULL,'69004','France','empFirstName2'); -insert into app.orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000','1996-08-16 00:00:00.000','1996-07-29 00:00:00.000',13,55.09,'Ottilies Kaoseladen','Mehrheimerstr. 369','Kaqln',NULL,'50739','Germany','empFirstName4'); -insert into app.orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000','1996-08-22 00:00:00.000','1996-08-12 00:00:00.000',18,55.28,'Blondel pare et fils 24', 'place Klacber','Strasbourg',NULL,'67000','France','empFirstName5'); +insert into orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuailles en stock 2','rue du Commerce','Lyon',NULL,'69004','France','empFirstName2'); +insert into orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000','1996-08-16 00:00:00.000','1996-07-29 00:00:00.000',13,55.09,'Ottilies Kaoseladen','Mehrheimerstr. 369','Kaqln',NULL,'50739','Germany','empFirstName4'); +insert into orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000','1996-08-22 00:00:00.000','1996-08-12 00:00:00.000',18,55.28,'Blondel pare et fils 24', 'place Klacber','Strasbourg',NULL,'67000','France','empFirstName5'); SELECT * FROM orders where FirstName='empFirstName2'; SELECT * FROM orders where FirstName='empFirstName4'; @@ -22,9 +22,9 @@ ALTER TABLE order_details ADD CustomerID String; SELECT count(*) FROM order_details; -insert into app.order_details values(10250,72,34.8,5,0,'custID4'); -insert into app.order_details values(10249,42,9.8,10,0,'custID14'); -insert into app.order_details values(10253,41,7.7,10,0,'custID7'); +insert into order_details values(10250,72,34.8,5,0,'custID4'); +insert into order_details values(10249,42,9.8,10,0,'custID14'); +insert into order_details values(10253,41,7.7,10,0,'custID7'); SELECT * FROM order_details where CustomerID='custID4'; SELECT * FROM order_details where CustomerID='custID14'; diff --git a/dtests/src/resources/scripts/diskFullTests/createAndLoadColumnTable.sql b/dtests/src/resources/scripts/diskFullTests/createAndLoadColumnTable.sql index ad444ad88e..2d38fe6354 100644 --- a/dtests/src/resources/scripts/diskFullTests/createAndLoadColumnTable.sql +++ b/dtests/src/resources/scripts/diskFullTests/createAndLoadColumnTable.sql @@ -8,7 +8,7 @@ CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL, VLD_FRM_DT TIMESTAMP NOT NULL, VLD_TO_DT TIMESTAMP, SRC_SYS_REF_ID VARCHAR(200) NOT NULL, - SRC_SYS_REC_ID VARCHAR(200)) USING column OPTIONS(partition_by 'AGREE_ID', buckets '32',key_columns 'CLIENT_ID,AGREE_ID ' ); + SRC_SYS_REC_ID VARCHAR(200)) USING column OPTIONS(partition_by 'AGREE_ID', buckets '32',key_columns 'CLIENT_ID,AGREE_ID ',redundancy '1' ); INSERT into AGREEMENT select id,abs(rand()*1000),abs(rand()*1000),'agree_cd','description','2018-01-01','2019-01-01',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' FROM range(10000000); CREATE TABLE BANK( @@ -21,5 +21,5 @@ CREATE TABLE BANK( VLD_FRM_DT TIMESTAMP NOT NULL, VLD_TO_DT TIMESTAMP, SRC_SYS_REF_ID VARCHAR(10) NOT NULL, - SRC_SYS_REC_ID VARCHAR(150)) USING column OPTIONS(partition_by 'BNK_ORG_ID', buckets '32',key_columns 'CLIENT_ID,BNK_ORG_ID,BNK_ID ' ) ; + SRC_SYS_REC_ID VARCHAR(150)) USING column OPTIONS(partition_by 'BNK_ORG_ID', buckets '32',key_columns 'CLIENT_ID,BNK_ORG_ID,BNK_ID ',redundancy '1' ) ; INSERT into BANK select id,id,abs(rand()*1000),abs(rand()*1000),'BNK_FULL_NM','RTNG_NUM',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' from range(4000000); diff --git a/dtests/src/resources/scripts/diskFullTests/createAndLoadPartitionedRowTable.sql b/dtests/src/resources/scripts/diskFullTests/createAndLoadPartitionedRowTable.sql index 618b827295..a2147ce973 100644 --- a/dtests/src/resources/scripts/diskFullTests/createAndLoadPartitionedRowTable.sql +++ b/dtests/src/resources/scripts/diskFullTests/createAndLoadPartitionedRowTable.sql @@ -1,4 +1,4 @@ -CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL, +CREATE TABLE AGREEMENTrow( AGREE_ID BIGINT NOT NULL, VER BIGINT NOT NULL, CLIENT_ID BIGINT NOT NULL, AGREE_CD VARCHAR(200), @@ -8,4 +8,4 @@ CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL, VLD_FRM_DT TIMESTAMP NOT NULL, VLD_TO_DT TIMESTAMP, SRC_SYS_REF_ID VARCHAR(200) NOT NULL, - SRC_SYS_REC_ID VARCHAR(200)) USING row OPTIONS(partition_by 'AGREE_ID', buckets '32'); \ No newline at end of file + SRC_SYS_REC_ID VARCHAR(200)) USING row OPTIONS(partition_by 'AGREE_ID', buckets '32',redundancy '1'); \ No newline at end of file diff --git a/dtests/src/resources/scripts/diskFullTests/createAndLoadReplicatedRowTable.sql b/dtests/src/resources/scripts/diskFullTests/createAndLoadReplicatedRowTable.sql index 4d7970f293..5410c9b4f3 100644 --- a/dtests/src/resources/scripts/diskFullTests/createAndLoadReplicatedRowTable.sql +++ b/dtests/src/resources/scripts/diskFullTests/createAndLoadReplicatedRowTable.sql @@ -1,4 +1,4 @@ -CREATE TABLE AGREEMENT( AGREE_ID BIGINT NOT NULL, +CREATE TABLE AGREEMENTrr( AGREE_ID BIGINT NOT NULL, VER BIGINT NOT NULL, CLIENT_ID BIGINT NOT NULL, AGREE_CD VARCHAR(200), diff --git a/dtests/src/resources/scripts/diskFullTests/insertQuery.sql b/dtests/src/resources/scripts/diskFullTests/insertQuery.sql index 0c52e459fd..6a17989c5a 100644 --- a/dtests/src/resources/scripts/diskFullTests/insertQuery.sql +++ b/dtests/src/resources/scripts/diskFullTests/insertQuery.sql @@ -1,2 +1,2 @@ -INSERT into AGREEMENT select id,abs(rand()*1000),abs(rand()*1000),'agree_cd','description','2018-01-01','2019-01-01',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' FROM range(2000000); -INSERT into BANK select id,id,abs(rand()*1000),abs(rand()*1000),'BNK_FULL_NM','RTNG_NUM',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' from range(2000000); +INSERT into AGREEMENT select id,abs(rand()*1000),abs(rand()*1000),'agree_cd','description','2018-01-01','2019-01-01',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' FROM range(200000000); +INSERT into BANK select id,id,abs(rand()*1000),abs(rand()*1000),'BNK_FULL_NM','RTNG_NUM',from_unixtime(unix_timestamp('2018-01-01 01:00:00')+floor(rand()*31536000)),from_unixtime(unix_timestamp('2019-01-01 01:00:00')+floor(rand()*31536000)),'src_sys_ref_id','src_sys_rec_id' from range(5000000); diff --git a/dtests/src/resources/scripts/security/dmlExtTblQuery.sql b/dtests/src/resources/scripts/security/dmlExtTblQuery.sql new file mode 100644 index 0000000000..ba96adf79c --- /dev/null +++ b/dtests/src/resources/scripts/security/dmlExtTblQuery.sql @@ -0,0 +1,10 @@ +SELECT count(*) FROM gemfire2.staging_employees; +SELECT * FROM gemfire2.staging_employees limit 5; +SELECT count(*) FROM gemfire2.staging_categories; +SELECT * FROM gemfire2.staging_categories; +SELECT City, COUNT(EmployeeID) AS NumEmployees FROM gemfire2.staging_employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees; +SELECT * FROM gemfire2.staging_customers; +SELECT * from gemfire2.staging_suppliers; +SELECT * FROM gemfire2.staging_orders limit 5; +SELECT count(*) FROM gemfire2.staging_orders; +SELECT ProductID, AVG(UnitPrice) AS AveragePrice FROM gemfire2.staging_products GROUP BY ProductID HAVING AVG(UnitPrice) > 70 ORDER BY AveragePrice; \ No newline at end of file diff --git a/dtests/src/resources/scripts/security/dmlRowColQuery.sql b/dtests/src/resources/scripts/security/dmlRowColQuery.sql index d2fc2d670f..d0219e7962 100644 --- a/dtests/src/resources/scripts/security/dmlRowColQuery.sql +++ b/dtests/src/resources/scripts/security/dmlRowColQuery.sql @@ -1,22 +1,22 @@ -SELECT count(*) FROM user2.employees; -SELECT * FROM user2.employees limit 5; -SELECT count(*) FROM user2.categories; -SELECT * FROM user2.categories; -SELECT City, COUNT(EmployeeID) AS NumEmployees FROM user2.employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees; -INSERT INTO user2.employees values(2,'empFirstName2','empLastName2','VicePresidentSales','Mrs.','1952-02-19 00:00:00.000','1992-08-14 00:00:00.000','908W.CapitalWay','Tacoma','WA','98401','USA','(206)555-9482',3457,'000','StudiedMBA.',NULL,'http://accweb/emmployees/fuller.bmp'); -INSERT INTO user2.categories values(10,'Pulses','dalandsprouts','00'); -UPDATE user2.employees SET LastName = 'last name' WHERE EmployeeID = 7; -DELETE FROM user2.employees WHERE LastName = 'last name' AND EmployeeID = 7; -UPDATE user2.categories SET CategoryName = 'Sweet Candies' WHERE CategoryId = 10; -DELETE FROM user2.categories WHERE categoryId = 10; -SELECT * FROM user2.customers; -SELECT * from user2.suppliers; -SELECT * FROM user2.orders limit 5; -SELECT count(*) FROM user2.orders; -SELECT ProductID, AVG(UnitPrice) AS AveragePrice FROM user2.products GROUP BY ProductID HAVING AVG(UnitPrice) > 70 ORDER BY AveragePrice; -INSERT into user2.customers values('custID4','abc','custName7','MarketingManager','aasdfasdfasd7','Strasbourg',NULL,'67000','France','88.60.15.31','88.60.15.32'); -INSERT into user2.order_details values(10253,41,7.7,10,0); -INSERT into user2.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuaillesenstock2','rueduCommerce','Lyon',NULL,'69004','France'); -UPDATE user2.customers SET companyName = 'New Company' WHERE CustomerId = 'ANTON'; -UPDATE user2.orders SET ShipCountry = 'INDIA' WHERE OrderID = 10250; -DELETE FROM user2.orders WHERE ShipCountry = 'INDIA' AND OrderID = 10250; \ No newline at end of file +SELECT count(*) FROM gemfire2.employees; +SELECT * FROM gemfire2.employees limit 5; +SELECT count(*) FROM gemfire2.categories; +SELECT * FROM gemfire2.categories; +SELECT City, COUNT(EmployeeID) AS NumEmployees FROM gemfire2.employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees; +INSERT INTO gemfire2.employees values(2,'empFirstName2','empLastName2','VicePresidentSales','Mrs.','1952-02-19 00:00:00.000','1992-08-14 00:00:00.000','908W.CapitalWay','Tacoma','WA','98401','USA','(206)555-9482',3457,'000','StudiedMBA.',NULL,'http://accweb/emmployees/fuller.bmp'); +INSERT INTO gemfire2.categories values(10,'Pulses','dalandsprouts','00'); +UPDATE gemfire2.employees SET LastName = 'last name' WHERE EmployeeID = 7; +DELETE FROM gemfire2.employees WHERE LastName = 'last name' AND EmployeeID = 7; +UPDATE gemfire2.categories SET CategoryName = 'Sweet Candies' WHERE CategoryId = 10; +DELETE FROM gemfire2.categories WHERE categoryId = 10; +SELECT * FROM gemfire2.customers; +SELECT * from gemfire2.suppliers; +SELECT * FROM gemfire2.orders limit 5; +SELECT count(*) FROM gemfire2.orders; +SELECT ProductID, AVG(UnitPrice) AS AveragePrice FROM gemfire2.products GROUP BY ProductID HAVING AVG(UnitPrice) > 70 ORDER BY AveragePrice; +INSERT into gemfire2.customers values('custID4','abc','custName7','MarketingManager','aasdfasdfasd7','Strasbourg',NULL,'67000','France','88.60.15.31','88.60.15.32'); +INSERT into gemfire2.order_details values(10253,41,7.7,10,0); +INSERT into gemfire2.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000','1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuaillesenstock2','rueduCommerce','Lyon',NULL,'69004','France'); +UPDATE gemfire2.customers SET companyName = 'New Company' WHERE CustomerId = 'ANTON'; +UPDATE gemfire2.orders SET ShipCountry = 'INDIA' WHERE OrderID = 10250; +DELETE FROM gemfire2.orders WHERE ShipCountry = 'INDIA' AND OrderID = 10250; \ No newline at end of file diff --git a/dtests/src/resources/scripts/security/user3Credentials.txt b/dtests/src/resources/scripts/security/user3Credentials.txt index a9269521b6..61c078918f 100644 --- a/dtests/src/resources/scripts/security/user3Credentials.txt +++ b/dtests/src/resources/scripts/security/user3Credentials.txt @@ -1 +1 @@ --u user3:user3123 \ No newline at end of file +-u gemfire3:gemfire3 diff --git a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/CDCIngestionApp.java b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/CDCIngestionApp.java index fef9e933cd..aa03d38ac7 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/CDCIngestionApp.java +++ b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/CDCIngestionApp.java @@ -27,14 +27,16 @@ public class CDCIngestionApp implements Runnable { private int startRange; private int endRange; private String endPoint; + private boolean isSecurityEnabled; - public CDCIngestionApp(int sRange, int eRange, int i, String path, String sqlServerInst, String hostName){ + public CDCIngestionApp(int sRange, int eRange, int i, String path, String sqlServerInst, String hostName, boolean isSecurity){ threadName = "Thread-" + i; startRange = sRange; endRange = eRange; filePath = path + "/insert" + i + ".sql"; sqlServer = sqlServerInst; endPoint = hostName; + isSecurityEnabled = isSecurity; } public void run() { @@ -57,9 +59,14 @@ public Connection getSnappyConnection() { Connection conn = null; String url = "jdbc:snappydata://" + endPoint; String driver = "io.snappydata.jdbc.ClientDriver"; - try { + Properties props = new Properties(); + if(isSecurityEnabled){ + props.setProperty("user","gemfire"); + props.setProperty("password","gemfire"); + } + try { Class.forName(driver); - conn = DriverManager.getConnection(url); + conn = DriverManager.getConnection(url,props); } catch (Exception ex) { System.out.println("Caught exception in getSnappyConnection() method" + ex.getMessage()); } @@ -74,7 +81,7 @@ public Connection getSqlServerConnection() { Class.forName(driver); String url; if (sqlServer.equals("sqlServer1")) { - url = "jdbc:sqlserver://sqlent.westus.cloudapp.azure.com:1433"; + url = "jdbc:sqlserver://sqlserver-ent16.copfedn1qbcz.us-west-2.rds.amazonaws.com:1433"; } else url = "jdbc:sqlserver://sqlserver2-et16.copfedn1qbcz.us-west-2.rds.amazonaws.com:1435"; String username = "sqldb"; @@ -93,69 +100,69 @@ public Connection getSqlServerConnection() { return conn; } -public void insertData(ArrayList queryArray, Connection conn) { + public void insertData(ArrayList queryArray, Connection conn) { PreparedStatement ps = null; - try { - final int batchSize = 1000; - int count = 0; - Random rnd = new Random(); - for (int i = 0; i < queryArray.size(); i++) { - String qStr = queryArray.get(i); - System.out.println("Query = " + qStr); - System.out.println("The startRange = " + startRange + " the endRange = " + endRange); - if (qStr.contains("PUT INTO")) { - for (int j = startRange; j <= endRange; j++) { - String newStr; - if (qStr.contains("?")) - newStr = qStr.replace("?", Integer.toString(j)); - else - newStr = qStr; - System.out.println("The new query String is " + newStr); - conn.createStatement().execute(newStr); - updateData(queryArray,conn); - } - } else if (qStr.contains("UPDATE")) { - ps = conn.prepareStatement(qStr); - int updateKey = rnd.nextInt(startRange); - ps.setInt(1, updateKey); - ps.execute(); - System.out.println("Key to be updated is " + updateKey); - } else if (qStr.contains("DELETE")) { - ps = conn.prepareStatement(qStr); - int delKey = rnd.nextInt(startRange); - ps.setInt(1, delKey); - ps.execute(); - System.out.println("Key to be deleted is " + delKey); - } else if(qStr.contains("INSERT INTO")){ - ps = conn.prepareStatement(qStr); - for (int j = startRange; j <= endRange; j++) { - int KEY_ID = j; - ps.setInt(1, KEY_ID); - ps.addBatch(); - if (++count % batchSize == 0) { - ps.executeBatch(); + try { + final int batchSize = 1000; + int count = 0; + Random rnd = new Random(); + for (int i = 0; i < queryArray.size(); i++) { + String qStr = queryArray.get(i); + System.out.println("Query = " + qStr); + System.out.println("The startRange = " + startRange + " the endRange = " + endRange); + if (qStr.contains("PUT INTO")) { + for (int j = startRange; j <= endRange; j++) { + String newStr; + if (qStr.contains("?")) + newStr = qStr.replace("?", Integer.toString(j)); + else + newStr = qStr; + System.out.println("The new query String is " + newStr); + conn.createStatement().execute(newStr); + updateData(queryArray, conn); } - } - System.out.println("Thread " + threadName + " finished ingesting " + (endRange - startRange) + " rows in a table"); + } else if (qStr.contains("UPDATE")) { + ps = conn.prepareStatement(qStr); + int updateKey = rnd.nextInt(startRange); + ps.setInt(1, updateKey); + ps.execute(); + System.out.println("Key to be updated is " + updateKey); + } else if (qStr.contains("DELETE")) { + ps = conn.prepareStatement(qStr); + int delKey = rnd.nextInt(startRange); + ps.setInt(1, delKey); + ps.execute(); + System.out.println("Key to be deleted is " + delKey); + } else if (qStr.contains("INSERT INTO")) { + ps = conn.prepareStatement(qStr); + for (int j = startRange; j <= endRange; j++) { + int KEY_ID = j; + ps.setInt(1, KEY_ID); + ps.addBatch(); + if (++count % batchSize == 0) { + ps.executeBatch(); + } + } + System.out.println("Thread " + threadName + " finished ingesting " + (endRange - startRange) + " rows in a table"); + } else + conn.createStatement().execute(qStr); + } + System.out.println("FINISHED: Thread " + threadName + " finished ingestion in all the tables"); + } catch (Exception e) { + System.out.println("Caught exception " + e.getMessage()); + } finally { + if (ps != null) try { + ps.close(); + } catch (SQLException ex) { + } + if (conn != null) try { + conn.close(); + } catch (SQLException ex) { } - else - conn.createStatement().execute(qStr); - } - System.out.println("FINISHED: Thread " + threadName + " finished ingestion in all the tables"); - } catch (Exception e) { - System.out.println("Caught exception " + e.getMessage()); - } finally { - if (ps != null) try { - ps.close(); - } catch (SQLException ex) { - } - if (conn != null) try { - conn.close(); - } catch (SQLException ex) { - } + } } -} + public void updateData(ArrayList queryArray, Connection conn) { try { for (int i = 0; i < queryArray.size(); i++) { @@ -210,12 +217,12 @@ public void start() { } } - public static void runIngestionApp(int sRange, int eRange, int thnCnt, String path, String sqlServerInst, String hostName) { + public static void runIngestionApp(int sRange, int eRange, int thnCnt, String path, String sqlServerInst, String hostName, boolean isSecurity) { ExecutorService executor = Executors.newFixedThreadPool(thnCnt); for (int i = 1; i <= thnCnt; i++) { String threadName = "Thread-" + i; System.out.println("Creating " + threadName); - executor.execute(new CDCIngestionApp(sRange, eRange, i,path, sqlServerInst, hostName)); + executor.execute(new CDCIngestionApp(sRange, eRange, i,path, sqlServerInst, hostName, isSecurity)); } executor.shutdown(); try { @@ -235,7 +242,7 @@ public static void main(String args[]) { String sqlServerInstance = args[4]; String hostname = args[5]; System.out.println("The startRange is " + sRange + " and the endRange is " + eRange); - runIngestionApp(sRange, eRange, threadCnt, insertQPAth, sqlServerInstance, hostname); + runIngestionApp(sRange, eRange, threadCnt, insertQPAth, sqlServerInstance, hostname, false); } catch (Exception e) { System.out.println("Caught exception in main " + e.getMessage()); } finally { diff --git a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java index cec434fc20..f6f9b3145b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/SnappyCDCTest.java @@ -108,9 +108,14 @@ public static void HydraTask_performRebalance() { public void performRebalance() { try { - Connection conn = SnappyTest.getLocatorConnection(); + Connection con = null; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if (isSecurityEnabled) + con = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + con = SnappyTest.getLocatorConnection(); Long startTime = System.currentTimeMillis(); - conn.createStatement().execute("call sys.rebalance_all_buckets();"); + con.createStatement().execute("call sys.rebalance_all_buckets();"); Long endTime = System.currentTimeMillis(); Long totalTime = endTime - startTime; Log.getLogWriter().info("The rebalance procedure took " + totalTime + " ms"); @@ -291,8 +296,13 @@ public void storeDataCount() { String fileName = SnappyCDCPrms.getDataLocation(); String schema = SnappyDataExtractorToolTestPrms.getSchemaName(); try { - Connection con = SnappyTest.getLocatorConnection(); - String tableCntQry = "SELECT COUNT(*) FROM SYS.SYSTABLES WHERE TABLESCHEMANAME='" +schema+ "' AND TABLENAME NOT LIKE 'SNAPPYSYS_INTERNA%'"; + Connection con = null; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if (isSecurityEnabled) + con = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + con = SnappyTest.getLocatorConnection(); + String tableCntQry = "SELECT COUNT(*) FROM SYS.SYSTABLES WHERE TABLESCHEMANAME='APP' AND TABLENAME NOT LIKE 'SNAPPYSYS_INTERNA%'"; ResultSet rs = con.createStatement().executeQuery(tableCntQry); while (rs.next()) tableCnt = rs.getInt(1); @@ -351,16 +361,22 @@ public void validateDataCount() { Map tableCntMap = (Map) SnappyBB.getBB().getSharedMap().get("tableCntMap"); Log.getLogWriter().info("tableCntMap size = " + tableCntMap.size() ); try { - Connection con = SnappyTest.getLocatorConnection(); + Connection con = null; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if (isSecurityEnabled) + con = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + con = SnappyTest.getLocatorConnection(); for (Map.Entry val : tableCntMap.entrySet()) { - long snappyCnt = 0; + int snappyCnt = 0; + String tableName = val.getKey(); long BBCnt = val.getValue(); String cntQry = "SELECT COUNT(*) FROM " + tableName; Log.getLogWriter().info("The query to be executed is " + cntQry); ResultSet rs3 = con.createStatement().executeQuery(cntQry); while (rs3.next()) - snappyCnt = rs3.getLong(1); + snappyCnt = rs3.getInt(1); rs3.close(); if (snappyCnt == BBCnt) Log.getLogWriter().info("SUCCESS : The cnt for table " + tableName + " = " + snappyCnt + " is EQUAL to the BB count = " + BBCnt); @@ -375,7 +391,7 @@ public void validateDataCount() { } } - public void getResultSet(Connection conn, Boolean isBeforeRestart, String fileName) { + public void getResultSet(Connection con, Boolean isBeforeRestart, String fileName) { SnappyDMLOpsUtil testInstance = new SnappyDMLOpsUtil(); String logFile = getCurrentDirPath()+ File.separator + "queryResultFiles"; File queryResultDir = new File(logFile); @@ -390,7 +406,7 @@ public void getResultSet(Connection conn, Boolean isBeforeRestart, String fileNa outputFile = logFile + File.separator + "afterRestartResultSet_query_" +System.currentTimeMillis() + "_" + i + ".out"; String qStr = queryList.get(i); Log.getLogWriter().info("The query for validation is " + qStr); - ResultSet snappyRS = conn.createStatement().executeQuery(qStr); + ResultSet snappyRS = con.createStatement().executeQuery(qStr); StructTypeImpl snappySti = ResultSetHelper.getStructType(snappyRS); List snappyList = ResultSetHelper.asList(snappyRS, snappySti, false); snappyRS.close(); @@ -526,7 +542,8 @@ public void runIngestionApp() { String queryPath = SnappyCDCPrms.getDataLocation(); String sqlServer = SnappyCDCPrms.getSqlServerInstance(); List endpoints = validateLocatorEndpointData(); - CDCIngestionApp.runIngestionApp(sRange, eRange, threadCnt, queryPath, sqlServer, endpoints.get(0)); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + CDCIngestionApp.runIngestionApp(sRange, eRange, threadCnt, queryPath, sqlServer, endpoints.get(0), isSecurityEnabled); } catch (Exception ex) { Log.getLogWriter().info("Caught Exception" + ex.getMessage() + " in runIngestionApp() method"); } @@ -552,7 +569,7 @@ public static void HydraTask_closeStreamingJob() { Log.getLogWriter().info("The curlCmd is " + curlCmd); pb = new ProcessBuilder("/bin/bash", "-c", curlCmd); log = new File("."); - String dest = log.getCanonicalPath() + File.separator + logFileName + "i"; + String dest = log.getCanonicalPath() + File.separator + logFileName + i; logFile = new File(dest); snappyTest.executeProcess(pb, logFile); } diff --git a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/cdcBasicValidationTest.conf b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/cdcBasicValidationTest.conf index 73f903671c..7fd6115746 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cdcConnector/cdcBasicValidationTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cdcConnector/cdcBasicValidationTest.conf @@ -9,7 +9,7 @@ These three operation will keep running for the time untill the totalTaskTimeSec And at the end validation is done ,to check if the changes in the sqlserver has been reflected in the snappy cluster."; INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; -INCLUDE $JTESTS/io/snappydata/hydra/cdcConnector/threadGroups.inc; +//INCLUDE $JTESTS/io/snappydata/hydra/cdcConnector/threadGroups.inc; io.snappydata.hydra.cdcConnector.SnappyCDCPrms-initStartRange = fcn (${initStartRange}) ncf; @@ -112,7 +112,7 @@ TASK taskClass = io.snappydata.hydra.cdcConnector.SnappyCDCTest taskMeth io.snappydata.hydra.cdcConnector.SnappyCDCPrms-threadCnt = ${threadCnt} io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = ${queryFilePath} io.snappydata.hydra.cdcConnector.SnappyCDCPrms-isPointLookUP = true - threadGroups = snappyTaskThread2 + threadGroups = locatorThreads startInterval = 300 //after every 5 mins endInterval = 300 maxThreads = 1; diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyAlterTablesTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyAlterTablesTest.java index e27e293e12..a1f619c622 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyAlterTablesTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyAlterTablesTest.java @@ -25,21 +25,26 @@ public class SnappyAlterTablesTest extends SnappyTest { public static void HydraTask_alterTables() throws SQLException { String query = null; - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); query = "ALTER TABLE orders ADD FirstName String"; executeUpdate(conn, query); query = "SELECT count(*) FROM orders"; executeQuery(conn, query); - query = "insert into app.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000'," + + query = "insert into orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000'," + "'1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34," + "'Victuailles en stock 2','rue du Commerce','Lyon',NULL,'69004','France'," + "'empFirstName2')"; executeUpdate(conn, query); - query = "insert into app.orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000'," + + query = "insert into orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000'," + "'1996-08-16 00:00:00.000','1996-07-29 00:00:00.000',13,55.09,'Ottilies Kaoseladen'," + "'Mehrheimerstr. 369','Kaqln',NULL,'50739','Germany','empFirstName4')"; executeUpdate(conn, query); - query = "insert into app.orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000'," + + query = "insert into orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000'," + "'1996-08-22 00:00:00.000','1996-08-12 00:00:00.000',18,55.28," + "'Blondel pare et fils 24', 'place Klacber','Strasbourg',NULL," + "'67000','France','empFirstName5')"; @@ -62,11 +67,11 @@ public static void HydraTask_alterTables() throws SQLException { executeUpdate(conn, query); query = "SELECT count(*) FROM order_details"; executeQuery(conn, query); - query = "insert into app.order_details values(10250,72,34.8,5,0,'custID4')"; + query = "insert into order_details values(10250,72,34.8,5,0,'custID4')"; executeUpdate(conn, query); - query = "insert into app.order_details values(10249,42,9.8,10,0,'custID14')"; + query = "insert into order_details values(10249,42,9.8,10,0,'custID14')"; executeUpdate(conn, query); - query = "insert into app.order_details values(10253,41,7.7,10,0,'custID7')"; + query = "insert into order_details values(10253,41,7.7,10,0,'custID7')"; executeUpdate(conn, query); query = "SELECT * FROM order_details where CustomerID='custID4'"; executeQuery(conn, query); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyColocationAndEvictionTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyColocationAndEvictionTest.java index c26aa26603..fef6180283 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyColocationAndEvictionTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyColocationAndEvictionTest.java @@ -51,10 +51,15 @@ public static synchronized void HydraTask_verifyEvictionLocalDestroy() throws SQ * Task to verify PR bucket local destroy eviction. */ public static synchronized void verifyEvictionLocalDestroy() throws SQLException { + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); Properties locatorProps = new Properties(); String locatorsList = SnappyTest.getLocatorsList("locators"); locatorProps.setProperty("locators", locatorsList); locatorProps.setProperty("mcast-port", "0"); + if (isSecurityEnabled) { + locatorProps.setProperty("user", "gemfire1"); + locatorProps.setProperty("password", "gemfire1"); + } Server server = ServerManager.getServerInstance(); server.start(locatorProps); Set regions = Misc.getGemFireCache().getAllRegions(); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyLocatorHATest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyLocatorHATest.java index 980b203ea9..b68841be65 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyLocatorHATest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyLocatorHATest.java @@ -130,7 +130,12 @@ public static void HydraTask_ddlOpAfterAllLocatorStop_ClusterRestart() { public static void HydraTask_validateTableDataOnClusterRestart() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); String query = "select count(*) from tab1"; ResultSet rs = conn.createStatement().executeQuery(query); long numRows = 0; diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyStartUpTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyStartUpTest.java index 020aab67f9..7166ab0530 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyStartUpTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyStartUpTest.java @@ -192,7 +192,11 @@ protected static void executeOps() { String query = "create table tab1 (id int, name String, address String) USING column " + "OPTIONS(partition_by 'id')"; try { - conn = getLocatorConnection(); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); conn.createStatement().executeUpdate(query); Log.getLogWriter().info("query executed successfully: " + query); query = "insert into tab1 values(111, 'aaa', 'hello')"; @@ -231,7 +235,11 @@ protected static void dropAndReCreateTable() { ResultSet rs; String query = "DROP TABLE IF EXISTS order_details"; try { - conn = getLocatorConnection(); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); conn.createStatement().executeUpdate(query); Log.getLogWriter().info("order_details table dropped successfully"); query = "DROP TABLE IF EXISTS staging_order_details"; @@ -401,7 +409,11 @@ public static void HydraTask_verifyTableData() { Vector tableNames, numRowsinTables = null; long expectedNumRows, actualNumRows = 0; try { - conn = getLocatorConnection(); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); tableNames = SnappyPrms.getTableList(); numRowsinTables = SnappyPrms.getNumRowsList(); if (tableNames.isEmpty() || numRowsinTables.isEmpty()) { diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java index 4c85e47602..997ea768f6 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java @@ -39,6 +39,8 @@ import io.snappydata.hydra.connectionPool.HikariConnectionPool; import io.snappydata.hydra.connectionPool.SnappyConnectionPoolPrms; import io.snappydata.hydra.connectionPool.TomcatConnectionPool; +import io.snappydata.hydra.security.SnappySecurityPrms; +import io.snappydata.hydra.security.SnappySecurityTest; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; @@ -51,7 +53,11 @@ import sql.SQLPrms; import sql.dmlStatements.DMLStmtIF; import sql.sqlutil.DMLStmtsFactory; -import util.*; +import util.PRObserver; +import util.StopStartPrms; +import util.StopStartVMs; +import util.TestException; +import util.TestHelper; import static hydra.Prms.maxResultWaitSec; @@ -94,6 +100,7 @@ public class SnappyTest implements Serializable { public static boolean isStopMode = TestConfig.tab().booleanAt(SnappyPrms.isStopMode, false); //default to false public static boolean forceStart = TestConfig.tab().booleanAt(SnappyPrms.forceStart, false); //default to false public static boolean forceCopy = TestConfig.tab().booleanAt(SnappyPrms.forceCopy, false); //default to false + public static boolean isSecurity = SnappySecurityPrms.getIsSecurity(); private static String primaryLocator = null; public static String leadHost = null; public static Long waitTimeBeforeStreamingJobStatus = TestConfig.tab().longAt(SnappyPrms.streamingJobExecutionTimeInMillis, 6000); @@ -180,9 +187,9 @@ public static synchronized void HydraTask_initializeSnappyTest() { snappyTest.generateConfig("primaryLocatorHost"); snappyTest.generateConfig("primaryLocatorPort"); } - - } + if (!SnappyBB.getBB().getSharedMap().containsKey("SECURITY_ENABLED")) + SnappyBB.getBB().getSharedMap().put("SECURITY_ENABLED", isSecurity); } } @@ -329,6 +336,13 @@ public static synchronized void HydraTask_generateSparkWorkerConfig() { } protected void generateNodeConfig(String logDir, boolean returnNodeLogDir) { + String secureBootProperties = " "; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) { + secureBootProperties = " -server-auth-provider=NONE " + SnappySecurityTest.getSecureBootProp(); + Log.getLogWriter().info(" secureBootProperties are " + secureBootProperties); + Log.getLogWriter().info("Security is Enabled"); + } if (logDirExists) return; String addr = HostHelper.getHostAddress(); int port = PortHelper.getRandomPort(); @@ -347,8 +361,7 @@ protected void generateNodeConfig(String logDir, boolean returnNodeLogDir) { while (locPort < 0 || locPort > 65535); nodeLogDir = HostHelper.getLocalHost() + " -dir=" + dirPath + clientPort + port + locPortString + locPort + SnappyPrms.getTimeStatistics() + SnappyPrms.getLogLevel() + - " -Dgemfire.DISALLOW_RESERVE_SPACE=true " + - " " + SnappyPrms.getLocatorLauncherProps(); + " " + SnappyPrms.getLocatorLauncherProps() + " " + secureBootProperties; SnappyBB.getBB().getSharedMap().put("locatorHost" + "_" + RemoteTestModule.getMyVmid(), HostHelper.getLocalHost()); SnappyBB.getBB().getSharedMap().put("locatorPort" + "_" + RemoteTestModule.getMyVmid(), @@ -385,8 +398,7 @@ protected void generateNodeConfig(String logDir, boolean returnNodeLogDir) { " -J-Dgemfire.CacheServerLauncher.SHUTDOWN_WAIT_TIME_MS=50000" + SnappyPrms.getFlightRecorderOptions(dirPath) + SnappyPrms.getGCOptions(dirPath) + " " + - " -Dgemfire.DISALLOW_RESERVE_SPACE=true " + - SnappyPrms.getServerLauncherProps() + + SnappyPrms.getServerLauncherProps() + " " + secureBootProperties + " -classpath=" + getStoreTestsJar(); Log.getLogWriter().info("Generated peer server endpoint: " + endpoint); SnappyBB.getBB().getSharedCounters().increment(SnappyBB.numServers); @@ -417,8 +429,7 @@ protected void generateNodeConfig(String logDir, boolean returnNodeLogDir) { " -J-Dgemfire.CacheServerLauncher.SHUTDOWN_WAIT_TIME_MS=50000" + SnappyPrms.getFlightRecorderOptions(dirPath) + SnappyPrms.getGCOptions(dirPath) + " " + - " -Dgemfire.DISALLOW_RESERVE_SPACE=true " + - SnappyPrms.getLeaderLauncherProps() + + SnappyPrms.getLeaderLauncherProps() + " " + secureBootProperties + " -spark.driver.extraClassPath=" + getStoreTestsJar() + " -spark.executor.extraClassPath=" + getStoreTestsJar(); try { @@ -560,11 +571,16 @@ public static void HydraTask_writeLocatorConfigData() { * @return Lead node status */ public static boolean isPrimaryLeadUpAndRunning() { + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); boolean isLeadNodeRunning = false; ResultSet rs; + Connection conn ; try { - Connection conn = getLocatorConnection(); - rs = conn.createStatement().executeQuery("select STATUS, PID, HOST from sys.members where KIND='primary lead'"); + if (isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); + rs = conn.createStatement().executeQuery("select STATUS, PID, HOST from sys.members where KIND='primary lead';"); while (rs.next()) { Log.getLogWriter().info("Checking the Lead node status..."); if (rs.getString("STATUS").equals("RUNNING")) { @@ -1267,6 +1283,17 @@ public static synchronized void setConnPoolType() { connPoolType = (int) SnappyBB.getBB().getSharedMap().get("connPoolType"); } + public static Connection getSecuredLocatorConnection(String usr, String pass) throws SQLException { + List endpoints = validateLocatorEndpointData(); + Properties props = new Properties(); + props.setProperty("user", usr); + props.setProperty("password", pass); + Connection conn = null; + String url = "jdbc:snappydata://" + endpoints.get(0) + "/"; + conn = getConnection(url, "io.snappydata.jdbc.ClientDriver", props); + return conn; + } + /** * Gets Client connection. */ @@ -1277,6 +1304,13 @@ public static Connection getLocatorConnection() throws SQLException { setConnPoolType(); else connPoolType = (int) SnappyBB.getBB().getSharedMap().get("connPoolType"); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + Properties props = new Properties(); + if(isSecurityEnabled) + { + props.setProperty("user","gemfire1"); + props.setProperty("password","gemfire1"); + } if (connPoolType == 0) { conn = HikariConnectionPool.getConnection(); @@ -1288,7 +1322,7 @@ public static Connection getLocatorConnection() throws SQLException { if (!runGemXDQuery) { String url = "jdbc:snappydata://" + endpoints.get(0); Log.getLogWriter().info("url is " + url); - conn = getConnection(url, "io.snappydata.jdbc.ClientDriver"); + conn = getConnection(url, "io.snappydata.jdbc.ClientDriver",props); } else { String url = "jdbc:gemfirexd://" + endpoints.get(0); Log.getLogWriter().info("url is " + url); @@ -1303,14 +1337,20 @@ public static Connection getLocatorConnectionUsingProps() throws List endpoints = validateLocatorEndpointData(); Properties props = new Properties(); Vector connPropsList = SnappyPrms.getConnPropsList(); + Connection conn = null; for (int i = 0; i < connPropsList.size(); i++) { String conProp = (String) connPropsList.elementAt(i); String conPropKey = conProp.substring(0, conProp.indexOf("=")); String conPropValue = conProp.substring(conProp.indexOf("=") + 1); props.setProperty(conPropKey, conPropValue); } - Connection conn = null; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); String url = "jdbc:snappydata://" + endpoints.get(0) + "/"; + if(isSecurityEnabled) + { + props.setProperty("user","gemfire1"); + props.setProperty("password","gemfire1"); + } conn = getConnection(url, "io.snappydata.jdbc.ClientDriver", props); return conn; } @@ -1318,13 +1358,23 @@ public static Connection getLocatorConnectionUsingProps() throws public static Connection getServerConnection() throws SQLException { List endpoints = validateServerEndpointData(); Connection conn = null; + Properties props = new Properties(); + props.setProperty("user","gemfire1"); + props.setProperty("password","gemfire1"); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); if (!runGemXDQuery) { String url = "jdbc:snappydata://" + endpoints.get(0); Log.getLogWriter().info("url is " + url); - conn = getConnection(url, "io.snappydata.jdbc.ClientDriver"); + if(isSecurityEnabled) + conn = getConnection(url, "io.snappydata.jdbc.ClientDriver", props); + else + conn = getConnection(url, "io.snappydata.jdbc.ClientDriver"); } else { String url = "jdbc:gemfirexd://" + endpoints.get(0); Log.getLogWriter().info("url is " + url); + if(isSecurityEnabled) + conn = getConnection(url, "io.snappydata.jdbc.ClientDriver", props); + else conn = getConnection(url, "io.snappydata.jdbc.ClientDriver"); } return conn; @@ -1641,6 +1691,7 @@ public static synchronized void HydraTask_executeSQLScripts() { maxPartitionSizeList = null, evictionByOptionList = null; File log = null, logFile = null; scriptNames = SnappyPrms.getSQLScriptNames(); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); if (scriptNames == null) { String s = "No Script names provided for executing in the Hydra TASK"; throw new TestException(s); @@ -1732,16 +1783,34 @@ public static synchronized void HydraTask_executeSQLScripts() { logFile = new File(dest); String primaryLocatorHost = getPrimaryLocatorHost(); String primaryLocatorPort = getPrimaryLocatorPort(); - ProcessBuilder pb = new ProcessBuilder(SnappyShellPath, "run", "-file=" + - filePath, "-param:dataLocation=" + dataLocation, - "-param:persistenceMode=" + persistenceMode, "-param:colocateWith=" + - colocateWith, "-param:partitionBy=" + partitionBy, - "-param:numPartitions=" + numPartitions, "-param:redundancy=" + - redundancy, "-param:recoverDelay=" + recoverDelay, - "-param:maxPartitionSize=" + maxPartitionSize, "-param:evictionByOption=" - + evictionByOption, "-client-port=" + primaryLocatorPort, - "-client-bind-address=" + primaryLocatorHost); - snappyTest.executeProcess(pb, logFile); + if (isSecurityEnabled) { + Log.getLogWriter().info("SP:Inside secure sqlScriptExecution"); + String user = "gemfire1"; + String pass = "gemfire1"; + ProcessBuilder pb = new ProcessBuilder(SnappyShellPath, "run", "-file=" + + filePath, "-param:dataLocation=" + dataLocation, + "-param:persistenceMode=" + persistenceMode, "-param:colocateWith=" + + colocateWith, "-param:partitionBy=" + partitionBy, + "-param:numPartitions=" + numPartitions, "-param:redundancy=" + + redundancy, "-param:recoverDelay=" + recoverDelay, + "-param:maxPartitionSize=" + maxPartitionSize, "-param:evictionByOption=" + + evictionByOption, "-client-port=" + primaryLocatorPort, + "-client-bind-address=" + primaryLocatorHost, "-user=" + user, "-password=" + pass); + + Log.getLogWriter().info(" cmd " + pb.command()); + snappyTest.executeProcess(pb, logFile); + } else { + ProcessBuilder pb = new ProcessBuilder(SnappyShellPath, "run", "-file=" + + filePath, "-param:dataLocation=" + dataLocation, + "-param:persistenceMode=" + persistenceMode, "-param:colocateWith=" + + colocateWith, "-param:partitionBy=" + partitionBy, + "-param:numPartitions=" + numPartitions, "-param:redundancy=" + + redundancy, "-param:recoverDelay=" + recoverDelay, + "-param:maxPartitionSize=" + maxPartitionSize, "-param:evictionByOption=" + + evictionByOption, "-client-port=" + primaryLocatorPort, + "-client-bind-address=" + primaryLocatorHost); + snappyTest.executeProcess(pb, logFile); + } } } catch (IOException e) { throw new TestException("IOException occurred while retriving destination logFile " + @@ -2005,6 +2074,18 @@ public static void HydraTask_executeSnappyStreamingJob() { "snappyStreamingJobTaskResult_" + System.currentTimeMillis() + ".log"); } + + public static void HydraTask_executeSnappyJobUsingJobScript() { + int count = 0; + int currentThread = snappyTest.getMyTid(); + String logFile = "snappyJobResult_thread_" + currentThread + "_" + System.currentTimeMillis() + ".log"; + SnappyBB.getBB().getSharedMap().put("logFilesForJobs_" + currentThread + "_" + System.currentTimeMillis(), logFile); + + snappyTest.executeSnappyJobUsingJobScript(SnappyPrms.getSnappyJobClassNames(), SnappyPrms.getUserAppName() + "_" + count + "_" + System.currentTimeMillis(), logFile); + count++; + } + + /** * Executes Snappy Jobs. */ @@ -2013,8 +2094,11 @@ public static synchronized void HydraTask_executeSnappyJob() { int currentThread = snappyTest.getMyTid(); String logFile = "snappyJobResult_thread_" + currentThread + "_" + System.currentTimeMillis() + ".log"; SnappyBB.getBB().getSharedMap().put("logFilesForJobs_" + currentThread + "_" + System.currentTimeMillis(), logFile); + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(!isSecurityEnabled) snappyTest.executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms .getUserAppJar(), jarPath, SnappyPrms.getUserAppName() + "_" + count + "_" + System.currentTimeMillis()); + else snappyTest.executeSnappyJobUsingJobScript(SnappyPrms.getSnappyJobClassNames(), SnappyPrms.getUserAppName() + "_" + count + "_" + System.currentTimeMillis(), logFile); count++; } @@ -2128,19 +2212,33 @@ protected void executeSnappyStreamingJobUsingJobScript(Vector jobClassNames, Str ProcessBuilder pb = null; File log = null; File logFile = null; + String confString=null; userAppJar = SnappyPrms.getUserAppJar(); snappyTest.verifyDataForJobExecution(jobClassNames, userAppJar); leadHost = getLeadHost(); String leadPort = (String) SnappyBB.getBB().getSharedMap().get("primaryLeadPort"); + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + String passFilePath = SnappySecurityPrms.getPassFile(); try { for (int i = 0; i < jobClassNames.size(); i++) { String userJob = (String) jobClassNames.elementAt(i); pb = new ProcessBuilder(snappyJobScript, "submit", "--lead", leadHost + ":" + leadPort, "--app-name", "myapp", "--class", userJob, "--app-jar", snappyTest.getUserAppJarLocation(userAppJar, jarPath), "--stream"); java.util.Map env = pb.environment(); - if (SnappyPrms.getCommaSepAPPProps() == null) { - env.put("APP_PROPS", "shufflePartitions=" + SnappyPrms.getShufflePartitions()); - } else { - env.put("APP_PROPS", SnappyPrms.getCommaSepAPPProps() + ",shufflePartitions=" + SnappyPrms.getShufflePartitions()); + if (SnappyPrms.getCommaSepAPPProps() != null) + confString = getConfForJobScripts(confString); + if (!isSecurityEnabled) { + if (SnappyPrms.getCommaSepAPPProps() == null) { + env.put("APP_PROPS", " --conf shufflePartitions=" + SnappyPrms.getShufflePartitions()); + } else { + env.put("APP_PROPS", confString + " --conf shufflePartitions=" + SnappyPrms.getShufflePartitions()); + } + } + else { + if (SnappyPrms.getCommaSepAPPProps() == null) { + env.put("APP_PROPS", " --conf shufflePartitions=" + SnappyPrms.getShufflePartitions() + " --passfile " + passFilePath); + } else { + env.put("APP_PROPS", SnappyPrms.getCommaSepAPPProps() + " --conf shufflePartitions=" + SnappyPrms.getShufflePartitions() + " --passfile " + passFilePath); + } } log = new File("."); String dest = log.getCanonicalPath() + File.separator + logFileName; @@ -2207,9 +2305,9 @@ protected void verifyDataForJobExecution(Vector jobClassNames, String userAppJar public void executeSnappyJob(Vector jobClassNames, String logFileName, String userAppJar, String jarPath, String appName) { + Log.getLogWriter().info("Inside executeSnappyJob"); String snappyJobScript = getScriptLocation("snappy-job.sh"); File log = null, logFile = null; -// userAppJar = SnappyPrms.getUserAppJar(); if (appName == null) appName = SnappyPrms.getUserAppName() + "_" + System.currentTimeMillis(); snappyTest.verifyDataForJobExecution(jobClassNames, userAppJar); leadHost = getLeadHost(); @@ -2296,9 +2394,19 @@ public String setCDCSparkAppCmds(String userAppArgs, String commonArgs, String s SnappyBB.getBB().getSharedMap().put("START_RANGE_APP2", finalEnd2 + 1); SnappyBB.getBB().getSharedMap().put("END_RANGE_APP2", finalEnd2 + 1000); } else if (appName.equals("CDCIngestionApp1")) { - userAppArgs = finalStart + " " + finalEnd + " " + userAppArgs; - SnappyBB.getBB().getSharedMap().put("finalStartRange", finalStart); - SnappyBB.getBB().getSharedMap().put("finalEndRange", finalEnd); + int BBfinalStart1 = (Integer) SnappyBB.getBB().getSharedMap().get("START_RANGE_APP1"); + int BBfinalEnd1 = (Integer) SnappyBB.getBB().getSharedMap().get("END_RANGE_APP1"); + int finalStart1, finalEnd1; + if (BBfinalStart1 == 0 || BBfinalEnd1 == 0) { + finalStart1 = finalStart; + finalEnd1 = finalEnd; + } else { + finalStart1 = BBfinalStart1; + finalEnd1 = BBfinalEnd1; + } + userAppArgs = finalStart1 + " " + finalEnd1 + " " + userAppArgs; + SnappyBB.getBB().getSharedMap().put("START_RANGE_APP1", finalEnd1 + 1); + SnappyBB.getBB().getSharedMap().put("END_RANGE_APP1", finalEnd1 + 1000); } else if (appName.equals("BulkDeleteApp")) { commonArgs = " --conf spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError" + " --conf spark.extraListeners=io.snappydata.hydra.SnappyCustomSparkListener "; @@ -2314,13 +2422,147 @@ public String setCDCSparkAppCmds(String userAppArgs, String commonArgs, String s return command; } + public void executeSnappyJobUsingJobScript(Vector jobClassNames, String userAppName, String logFileName) { + String snappyJobScript = getScriptLocation("snappy-job.sh"); + if (userAppName == null) userAppName = SnappyPrms.getUserAppName() + "_" + System.currentTimeMillis(); + ProcessBuilder pb = null; + File log = null; + File logFile = null; + String Parameters; + String confString = ""; + String[] confParameter = null; + userAppJar = SnappyPrms.getUserAppJar(); + snappyTest.verifyDataForJobExecution(jobClassNames, userAppJar); + leadHost = getLeadHost(); + String leadPort = getLeadPort(); + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + String passFilePath = SnappySecurityPrms.getPassFile(); + if (SnappyPrms.getCommaSepAPPProps() != null) + confString = getConfForJobScripts(confString); + if (SnappyPrms.useJDBCConnInSnappyJob()) { + String primaryLocatorHost = getPrimaryLocatorHost(); + String primaryLocatorPort = getPrimaryLocatorPort(); + confString += "--conf primaryLocatorHost=" + primaryLocatorHost + " --conf primaryLocatorPort=" + primaryLocatorPort; + Log.getLogWriter().info("SP: confString for useJDBCConnInSnappyJob : " + confString.toString()); + } + if (SnappyPrms.isLongRunningJob()) { + confString += " --conf maxResultWaitSec=" + maxResultWaitSecs; + } + if (SnappyPrms.hasDynamicAppProps()) { + String dmlProps = dynamicAppProps.get(getMyTid()); + if (dmlProps == null) { + dmlProps = dynamicAppProps.get(getMyTid()); + if (dmlProps == null) throw new TestException("Test issue: dml statement for " + + getMyTid() + " is null"); + } + ArrayList columnValues = new ArrayList<>(); + if (dmlProps.contains("\"")) { + String str3 = dmlProps; + while (str3.contains("\"")) { + int beginIndex = str3.indexOf("\""); + int endIndex = str3.indexOf("\"", beginIndex + 2); + String str1 = str3.substring(0, beginIndex); + columnValues.addAll(new ArrayList<>(Arrays.asList(str1.split(",")))); + String str2 = str3.substring(beginIndex, endIndex + 1); + str2 = columnValues.get((columnValues.size()) - 1) + str2; + columnValues.set((columnValues.size()) - 1, str2); + if (endIndex < str3.length()) + str3 = str3.substring(endIndex + 2, str3.length()); + else str3 = ""; + //columnValues.add(str2); + } + if (str3.length() > 0) { + columnValues.addAll(new ArrayList<>(Arrays.asList(str3.split(",")))); + } + } else { + columnValues = new ArrayList<>(Arrays.asList(dmlProps.split(","))); + } + if(columnValues.size() != 0){ + for (int i = 0; i < columnValues.size(); i++) { + String confValue = columnValues.get(i); + if(confValue.contains(" ") || confValue.contains("(") || confValue.contains(")") || confValue.contains("<") || confValue.contains("'") || confValue.contains(">")){ + confValue = confValue.replace(" ","\\ "); + confValue = confValue.replace("(","\\("); + confValue = confValue.replace(")","\\)"); + confValue = confValue.replace("<","\\<"); + confValue = confValue.replace(">","\\>"); + confValue = confValue.replace("'","\\'"); + } + Log.getLogWriter().info("SP: each config is " + confValue); + confString += " --conf " + confValue; + } + } + Log.getLogWriter().info("SP: confString for hasDynamicAppProps : " + confString); + } + if (isSecurityEnabled){ + if(!confString.contains("--passfile")) // check for security related tests + confString += " --passfile " + passFilePath + " --conf logFileName=" + logFileName; + else + confString += " --conf logFileName=" + logFileName; + } + + Log.getLogWriter().info("SP: confString for security: " + confString); + + try { + String cmd; + for (int i = 0; i < jobClassNames.size(); i++) { + String userJob = (String) jobClassNames.elementAt(i); + cmd = snappyJobScript + " submit --lead " + leadHost + ":" + leadPort + + " --app-name " + userAppName + " --class " + userJob + " --app-jar " + + snappyTest.getUserAppJarLocation(userAppJar, jarPath); + if (!confString.isEmpty()) + cmd += confString; + + pb = new ProcessBuilder("/bin/bash", "-c", cmd); + Log.getLogWriter().info("Command is : " + pb.command()); + + log = new File("."); + String dest = log.getCanonicalPath() + File.separator + logFileName; + logFile = new File(dest); + snappyTest.executeProcess(pb, logFile); + } + boolean retry = snappyTest.getSnappyJobsStatus(snappyJobScript, logFile, leadPort); + if (retry && jobSubmissionCount <= SnappyPrms.getRetryCountForJob()) { + jobSubmissionCount++; + Thread.sleep(180000); + Log.getLogWriter().info("Job failed due to primary lead node failover. Resubmitting" + + " the job to new primary lead node....."); + retrievePrimaryLeadHost(); + HydraTask_executeSnappyJob(); + } + } catch (IOException e) { + throw new TestException("IOException occurred while retriving destination logFile path " + + log + "\nError Message:" + e.getMessage()); + } catch (InterruptedException e) { + throw new TestException("Exception occurred while waiting for the snappy streaming job " + + "process re-execution." + "\nError Message:" + e.getMessage()); + } + // } + } + + public String getConfForJobScripts(String confString) { + String Parameters; + String[] confParameter = null; + Parameters = SnappyPrms.getCommaSepAPPProps(); + Log.getLogWriter().info("Parameters : " + Parameters); + confParameter = Parameters.split(","); + for (int i = 0; i < confParameter.length; i++) { + Log.getLogWriter().info("ConfParameter : " + confParameter[i].toString()); + confString += " --conf " + confParameter[i].toString(); + } + return confString; + } + + public void executeSparkJob(Vector jobClassNames, String logFileName) { String snappyJobScript = getScriptLocation("spark-submit"); boolean isCDCStream = SnappyCDCPrms.getIsCDCStream(); ProcessBuilder pb = null; File log = null, logFile = null; userAppJar = SnappyPrms.getUserAppJar(); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); snappyTest.verifyDataForJobExecution(jobClassNames, userAppJar); + String commonArgs = ""; try { for (int i = 0; i < jobClassNames.size(); i++) { String userJob = (String) jobClassNames.elementAt(i); @@ -2330,9 +2572,15 @@ public void executeSparkJob(Vector jobClassNames, String logFileName) { String primaryLocatorHost = getPrimaryLocatorHost(); String primaryLocatorPort = getPrimaryLocatorPort(); String userAppArgs = SnappyPrms.getUserAppArgs(); - String commonArgs = " --conf spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError" + + commonArgs = " --conf spark.executor.extraJavaOptions=-XX:+HeapDumpOnOutOfMemoryError" + " --conf spark.extraListeners=io.snappydata.hydra.SnappyCustomSparkListener " + " --conf snappydata.connection=" + primaryLocatorHost + ":" + primaryLocatorPort; + if (isSecurityEnabled) { + String sparkSubmitExtraPrms = SnappyPrms.getSparkSubmitExtraPrms(); + if (!sparkSubmitExtraPrms.contains("spark.snappydata.store.user")) //check for security tests + commonArgs = " --conf spark.snappydata.store.user=gemfire1 --conf spark.snappydata.store.password=gemfire1 " + commonArgs; + } + Log.getLogWriter().info("SP: The commonArgs are " + commonArgs); log = new File("."); String dest = log.getCanonicalPath() + File.separator + logFileName; logFile = new File(dest); @@ -2496,6 +2744,8 @@ protected void simulateStream() { public boolean getSnappyJobsStatus(String snappyJobScript, File logFile, String leadPort) { boolean found = false; try { + String passFilePath = SnappySecurityPrms.getPassFile(); + Log.getLogWriter().info("SP: passFilePath = " + passFilePath); String line = null; Set jobIds = new LinkedHashSet(); FileReader freader = new FileReader(logFile); @@ -2517,9 +2767,19 @@ public boolean getSnappyJobsStatus(String snappyJobScript, File logFile, String String dest = log.getCanonicalPath() + File.separator + "jobStatus_" + RemoteTestModule .getCurrentThread().getThreadId() + "_" + System.currentTimeMillis() + ".log"; File commandOutput = new File(dest); - String expression = snappyJobScript + " status --lead " + leadHost + ":" + leadPort + " " + - "--job-id " + str + " > " + commandOutput + " 2>&1 ; grep -e '\"status\": " + - "\"FINISHED\"' -e 'curl:' -e '\"status\": \"ERROR\"' " + commandOutput + " | wc -l)\""; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + String expression = null; + if(isSecurityEnabled) { + expression = snappyJobScript + " status --lead " + leadHost + ":" + leadPort + " " + + "--job-id " + str + " --passfile " + passFilePath + " > " + commandOutput + " 2>&1 ; grep -e '\"status\": " + + "\"FINISHED\"' -e 'curl:' -e '\"status\": \"ERROR\"' " + commandOutput + " | wc -l)\""; + } + else { + expression = snappyJobScript + " status --lead " + leadHost + ":" + leadPort + " " + + "--job-id " + str + " > " + commandOutput + " 2>&1 ; grep -e '\"status\": " + + "\"FINISHED\"' -e 'curl:' -e '\"status\": \"ERROR\"' " + commandOutput + " | wc -l)\""; + } + Log.getLogWriter().info("SP: The expression is " + expression); String command = "while [ \"$(" + expression + " -le 0 ] ; do rm " + commandOutput + " ; touch " + commandOutput + " ; sleep " + SnappyPrms.getSleepTimeSecsForJobStatus() + " ; done"; @@ -3728,13 +3988,19 @@ public List getHostNameFromConf(String nodeName) { protected void startSnappyLocator() { File log = null; ProcessBuilder pb = null; + String secureBootProperties = ""; List hostNames = getHostNameFromConf("locators"); try { + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) { + secureBootProperties = " -server-auth-provider=NONE " + SnappySecurityTest.getSecureBootProp(); + Log.getLogWriter().info("SP: secureBootProperties are " + secureBootProperties); + } if (useRowStore) { Log.getLogWriter().info("Starting locator/s using rowstore option..."); - pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-locators.sh"), "start", "rowstore"); + pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-locators.sh"), "start", "rowstore");//,secureBootProperties); } else { - pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-locators.sh"), "start"); + pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-locators.sh"), "start");//, secureBootProperties); } log = new File("."); String dest = log.getCanonicalPath() + File.separator + "snappyLocatorSystem.log"; @@ -3752,13 +4018,19 @@ protected void startSnappyLocator() { protected void startSnappyServer() { File log = null; ProcessBuilder pb = null; + String secureBootProperties = ""; List hostNames = getHostNameFromConf("servers"); try { + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) { + secureBootProperties = " -server-auth-provider=NONE " + SnappySecurityTest.getSecureBootProp(); + Log.getLogWriter().info("SP: secureBootProperties are " + secureBootProperties); + } if (useRowStore) { Log.getLogWriter().info("Starting server/s using rowstore option..."); - pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-servers.sh"), "start", "rowstore"); + pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-servers.sh"), "start", "rowstore");//, secureBootProperties); } else { - pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-servers.sh"), "start"); + pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-servers.sh"), "start");//, secureBootProperties); } log = new File("."); String dest = log.getCanonicalPath() + File.separator + "snappyServerSystem.log"; @@ -3789,10 +4061,16 @@ public static void sleepForMs(int sleepTimeInSec){ protected void startSnappyLead() { File log = null; + String secureBootProperties = ""; List hostNames = getHostNameFromConf("leads"); try { + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) { + secureBootProperties = " -server-auth-provider=NONE " + SnappySecurityTest.getSecureBootProp(); + Log.getLogWriter().info("SP: secureBootProperties are " + secureBootProperties); + } ProcessBuilder pb = new ProcessBuilder(snappyTest.getScriptLocation("snappy-leads.sh"), - "start"); + "start");//,secureBootProperties); log = new File("."); String dest = log.getCanonicalPath() + File.separator + "snappyLeaderSystem.log"; File logFile = new File(dest); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/commonPassFile.txt b/dtests/src/test/java/io/snappydata/hydra/cluster/commonPassFile.txt new file mode 100644 index 0000000000..58e4611e14 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/commonPassFile.txt @@ -0,0 +1 @@ +-u gemfire1:gemfire1 \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/commonProps.inc b/dtests/src/test/java/io/snappydata/hydra/cluster/commonProps.inc new file mode 100644 index 0000000000..5f717a0690 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/commonProps.inc @@ -0,0 +1,2 @@ +INCLUDE $JTESTS/io/snappydata/hydra/cluster/securityProps.inc; +//io.snappydata.hydra.cluster.SnappyPrms-columnBatchSize = 500; diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/securityProps.inc b/dtests/src/test/java/io/snappydata/hydra/cluster/securityProps.inc new file mode 100644 index 0000000000..aac124f65d --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/securityProps.inc @@ -0,0 +1,2 @@ +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = ONEOF true true FOENO; +io.snappydata.hydra.security.SnappySecurityPrms-passFile = /export/shared/QA_DATA/commonPassFile.txt; diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/startDualModeCluster.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/startDualModeCluster.conf index 0729821fae..f545fbfd1e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/startDualModeCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/startDualModeCluster.conf @@ -7,6 +7,7 @@ for long running test scenarios where in cluster will be started in first test a INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_4.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.GemFirePrms-names = gemfire1; hydra.ClientPrms-gemfireNames = gemfire1; diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/startEmbeddedModeClusterWithUserConfs.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/startEmbeddedModeClusterWithUserConfs.conf index a96e3f5ead..370586f96b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/startEmbeddedModeClusterWithUserConfs.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/startEmbeddedModeClusterWithUserConfs.conf @@ -5,6 +5,7 @@ hydra.Prms-testDescription = "This test starts the snappy embedded mode cluster INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_1.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; THREADGROUP snappyThread totalThreads = fcn "(${${A}Hosts} * ${${A}VMsPerHost} * ${${A}ThreadsPerVM}) " ncf diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/startSnappyCluster.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/startSnappyCluster.conf index c42138ff3f..6f97ab7a53 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/startSnappyCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/startSnappyCluster.conf @@ -3,6 +3,7 @@ hydra.Prms-testDescription = "This test starts the snappy embedded mode cluster. INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_3.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.gemfirexd.GfxdHelperPrms-persistDD = true; hydra.gemfirexd.GfxdHelperPrms-createDiskStore = true; @@ -40,6 +41,7 @@ THREADGROUP snappyThreads clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${B}\", ${${B}Hosts}, true)" ncf; + INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_initializeSnappyTest runMode = always threadGroups = snappyThreads, locatorThreads, snappyStoreThreads, leadThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededMode.conf b/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededMode.conf index 9c96ae13da..275bae85e2 100644 --- a/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededMode.conf +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededMode.conf @@ -5,6 +5,7 @@ hydra.Prms-testDescription = "This test starts the snappy cluster, then initiali INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_3.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.GemFirePrms-names = gemfire1; hydra.ClientPrms-gemfireNames = gemfire1; diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededModeCreateUsingScript.conf b/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededModeCreateUsingScript.conf index e989dc5ece..921b2cdfcb 100644 --- a/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededModeCreateUsingScript.conf +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctEmbededModeCreateUsingScript.conf @@ -5,6 +5,7 @@ hydra.Prms-testDescription = "This test starts the snappy cluster, then initiali INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_3.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.GemFirePrms-names = gemfire1; hydra.ClientPrms-gemfireNames = gemfire1; diff --git a/dtests/src/test/java/io/snappydata/hydra/dataExtractorTool/DataExtractorToolTest.java b/dtests/src/test/java/io/snappydata/hydra/dataExtractorTool/DataExtractorToolTest.java index 47ca509531..2835ad5635 100644 --- a/dtests/src/test/java/io/snappydata/hydra/dataExtractorTool/DataExtractorToolTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/dataExtractorTool/DataExtractorToolTest.java @@ -56,12 +56,16 @@ public static void HydraTask_executeQuery() { public void executeQuery() { int expectedExcptCnt = SnappySecurityPrms.getExpectedExcptCnt(); + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); Log.getLogWriter().info("SP: Inside runDMLOps "); String queryFile = SnappySecurityPrms.getDataLocation(); Connection conn = null; ArrayList queryArray = getQueryArr(queryFile); try { - conn = getLocatorConnection(); + if (isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); for (int i = 0; i < queryArray.size(); i++) { try { String queryStr = (String) queryArray.get(i); @@ -113,6 +117,7 @@ public static void HydraTask_ExtractData() { public void extractData() { Connection conn; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); Vector filePath = SnappyPrms.getDataLocationList(); String ddlPath = filePath.get(0).toString(); String dataPath = filePath.get(1).toString(); @@ -126,7 +131,10 @@ public void extractData() { try { ProcessBuilder p = new ProcessBuilder("/bin/bash", "-c", cmd); snappyTest.executeProcess(p, logFile); - conn = getLocatorConnection(); + if (isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); String query1 = "call sys.EXPORT_DATA('" + dataPath + "','parquet','all','true')"; conn.createStatement().execute(query1); String query2 = "call sys.EXPORT_DDLS('" + ddlPath + "')"; @@ -169,15 +177,18 @@ public void createTableFromExtractedDDLs() { String primaryLocatorPort = getPrimaryLocatorPort(); Log.getLogWriter().info("SP: The primaryLocatorHost = " + primaryLocatorHost + "\n The primaryLocatorPort is " + primaryLocatorPort); - + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + String userPwdStr=""; + if(isSecurityEnabled) + userPwdStr = " -user=gemfire1 , -password=gemfire1 "; //First drop the tables already in the cluster: ProcessBuilder pb1 = new ProcessBuilder(SnappyShellPath, "run", "-file=" + - dropTableQPath, "-client-port=" + primaryLocatorPort, "-client-bind-address=" + primaryLocatorHost); + dropTableQPath, "-client-port=" + primaryLocatorPort, "-client-bind-address=" + primaryLocatorHost + userPwdStr ); snappyTest.executeProcess(pb1, logFile); //Then Create the tables from extracted ddls. ProcessBuilder pb2 = new ProcessBuilder(SnappyShellPath, "run", "-file=" + - destPath, "-client-port=" + primaryLocatorPort, "-client-bind-address=" + primaryLocatorHost); + destPath, "-client-port=" + primaryLocatorPort, "-client-bind-address=" + primaryLocatorHost + userPwdStr); snappyTest.executeProcess(pb2, logFile); } catch (Exception ex) { throw new io.snappydata.test.util.TestException("Task createTableFromExtracteddDDL got exception " + ex.getMessage()); diff --git a/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTest.conf b/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTest.conf index 6a47cd9192..7d72762dc4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTest.conf @@ -24,8 +24,8 @@ TASK taskClass = io.snappydata.hydra.diskFullTests.SnappyDataDiskFullTe io.snappydata.hydra.cdcConnector.SnappyCDCPrms-snappyFileLoc = ${snappyFileLoc} maxThreads = 1 maxTimesToRun = 4 - startInterval = 60 - endInterval = 60 + startInterval = 20 + endInterval = 20 threadGroups = locatorThreads ; diff --git a/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTestWithDeletes.conf b/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTestWithDeletes.conf index 09656204fd..2a2aa1dc75 100644 --- a/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTestWithDeletes.conf +++ b/dtests/src/test/java/io/snappydata/hydra/diskFullTests/diskFullTestWithDeletes.conf @@ -15,7 +15,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H TASK taskClass = io.snappydata.hydra.diskFullTests.SnappyDataDiskFullTest taskMethod = HydraTask_executeFallocate io.snappydata.hydra.diskFullTests.SnappyDataDiskFullPrms-fileSize = ${fileSize} - io.snappydata.hydra.diskFullTests.SnappyDataDiskFullPrms-snappyFileLoc = ${snappyFileLoc} + io.snappydata.hydra.cdcConnector.SnappyCDCPrms-snappyFileLoc = ${snappyFileLoc} maxThreads = 1 maxTimesToRun = 5 startInterval = 2 diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/nwReplicatedRowTablesTest.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/nwReplicatedRowTablesTest.conf index a67608e577..f7ddb25ce1 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/nwReplicatedRowTablesTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/nwReplicatedRowTablesTest.conf @@ -8,7 +8,7 @@ sql script only executes the northwind queries on the tables created and loaded INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; -INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.northwind.CreateAndLoadNWTablesJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},createLargeOrderTable=${createLargeOrderTable}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf index 08e49b37fa..5307c4031e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf @@ -3,6 +3,7 @@ hydra.Prms-testDescription = "This test starts the snappy embedded as well as sp INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_4.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.GemFirePrms-names = gemfire1; hydra.ClientPrms-gemfireNames = gemfire1; diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/startEmbeddedModeCluster.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/startEmbeddedModeCluster.conf index e503b5fb4f..06318a943b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/startEmbeddedModeCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/startEmbeddedModeCluster.conf @@ -3,6 +3,7 @@ hydra.Prms-testDescription = "This test starts the snappy embedded mode cluster. INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_3.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.GemFirePrms-names = gemfire1; hydra.ClientPrms-gemfireNames = gemfire1; diff --git a/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.conf b/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.conf index 9e916d40d0..be14a13e78 100644 --- a/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.conf +++ b/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.conf @@ -10,6 +10,7 @@ The test verifies that each time, the cluster is restarted properly and all the INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_4.inc; +INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.gemfirexd.GfxdHelperPrms-persistDD = true; hydra.gemfirexd.GfxdHelperPrms-createDiskStore = true; diff --git a/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityPrms.java b/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityPrms.java index e6fb3d6ca9..f7a08954b2 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityPrms.java +++ b/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityPrms.java @@ -40,6 +40,8 @@ public class SnappySecurityPrms extends SnappyPrms{ public static Long isGrant; + public static Long isExtTblSecurity; + public static Long isRevoke; public static Long isJoinQuery; @@ -54,6 +56,36 @@ public class SnappySecurityPrms extends SnappyPrms{ public static Long isSecurity; + public static Long passFile; + + public static Long isAltTableRLS; + + public static Long isRLS; + + public static Long isDropPolicy; + + public static Long numOfPolicy; + + public static int getPolicyCnt() { + Long key = numOfPolicy; + return tasktab().intAt(key, tab().intAt(key, 1)); + } + + public static boolean getIsAltTableRLS() { + Long key = isAltTableRLS; + return tasktab().booleanAt(key, tab().booleanAt(key, false)); + } + + public static boolean getIsRLS() { + Long key = isRLS; + return tasktab().booleanAt(key, tab().booleanAt(key, false)); + } + + public static boolean getIsDropPolicy() { + Long key = isDropPolicy; + return tasktab().booleanAt(key, tab().booleanAt(key, false)); + } + public static Vector getDmlOps() { Long key = dmlOperations; return BasePrms.tasktab().vecAt(key, BasePrms.tab().vecAt(key, null)); @@ -66,12 +98,24 @@ public static String getDataLocation(){ return dataLoc; } + public static String getPassFile(){ + String pwdFile = tasktab().stringAt(passFile, tab().stringAt + (passFile, null)); + if (pwdFile == null) return ""; + return pwdFile; + } + public static Vector getSchema() { Long key = onSchema; return BasePrms.tasktab().vecAt(key, BasePrms.tab().vecAt(key, null)); } - public static boolean getIsGrant() { + public static boolean getIsExtTblSecurity() { + Long key = isExtTblSecurity; + return tasktab().booleanAt(key, tab().booleanAt(key, false)); + } + + public static boolean getIsGrant() { Long key = isGrant; return tasktab().booleanAt(key, tab().booleanAt(key, false)); } @@ -81,6 +125,11 @@ public static boolean getIsRevoke() { return tasktab().booleanAt(key, tab().booleanAt(key, false)); } + public static boolean getIsSecurity() { + Long key = isSecurity; + return tasktab().booleanAt(key, tab().booleanAt(key, false)); + } + public static boolean getIsJoinQuery() { Long key = isJoinQuery; return tasktab().booleanAt(key, tab().booleanAt(key, false)); diff --git a/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityTest.java b/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityTest.java index 2b14ffcc0d..8ce384ad5e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/security/SnappySecurityTest.java @@ -18,33 +18,111 @@ import hydra.Log; import hydra.RemoteTestModule; +import io.snappydata.hydra.cdcConnector.SnappyCDCPrms; import io.snappydata.hydra.cluster.SnappyPrms; import io.snappydata.hydra.cluster.SnappyTest; import javolution.io.Struct; import util.TestException; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.BufferedReader; import java.io.FileReader; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; +import java.io.*; +import java.sql.*; + + public class SnappySecurityTest extends SnappyTest { private static Integer expectedExceptionCnt = 0; private static Integer unExpectedExceptionCnt = 0; public static Boolean isAuthorized = false; - public static String adminUser = "user1"; - public static String unAuthUser = "user5"; + public static String adminUser = "gemfire1"; + public static String unAuthUser = "gemfire5"; + public static SnappySecurityTest snappySecurityTest; + public static String secureBootProp = ""; + private static HashMap, String> policySelectQueryMap = new HashMap<>(); + private static HashMap, String> policyFullSelectQueryMap = new HashMap<>(); + private static HashMap policyUserMap = new HashMap<>(); public static void HydraTask_runQuery() throws SQLException { - runQuery(); + Boolean isRLS = SnappySecurityPrms.getIsRLS(); + if (isRLS) + validateQuery(); + else + runQuery(); + } + + public static void HydraTask_startLdapServer() { + if (snappySecurityTest == null) { + snappySecurityTest = new SnappySecurityTest(); + } + snappySecurityTest.startLdapServer(); + } + + public void startLdapServer() { + String ldapScriptPath = SnappyCDCPrms.getDataLocation(); + if (snappyTest == null) { + snappyTest = new SnappyTest(); + } + try { + String dest = getCurrentDirPath() + File.separator + "ldapServerStart.log"; +// String propFile = getCurrentDirPath() + File.separator + "../../../secureBootProp.log"; + String propFile = "/export/shared/QA_DATA/secureBootProp.log"; + File ldapServerFile = new File(dest); + File secureBootPropFile = new File(propFile); + if(secureBootPropFile.exists()) + { + secureBootPropFile.delete(); + secureBootPropFile.createNewFile(); + } + if (!ldapServerFile.exists()) { + String cmd = "nohup " + ldapScriptPath + "/start-ldap-server.sh " + ldapScriptPath + "/auth.ldif > " + ldapServerFile + " & "; + ProcessBuilder pb = new ProcessBuilder("/bin/bash", "-c", cmd); + snappyTest.executeProcess(pb, ldapServerFile); + } + Thread.sleep(30000); //wait until the ldapserver starts. + + String cmd1 = "grep Boot " + dest; + ProcessBuilder pb1 = new ProcessBuilder("/bin/bash", "-c", cmd1); + snappyTest.executeProcess(pb1, secureBootPropFile); + } + catch (InterruptedException ex) { + Log.getLogWriter().info("Caught InterruptedException in startLdapServer method " + ex.getMessage()); + } + catch (IOException io) { + Log.getLogWriter().info("Caught IO exception in startLdapServer method " + io.getMessage()); + } + } + + public static String getSecureBootProp() { + try { + File log = new File("."); + // String propFile = log.getCanonicalPath() + File.separator + "../../../secureBootProp.log"; + String propFile = "/export/shared/QA_DATA/secureBootProp.log"; + File secureBootPropFile = new File(propFile); + FileInputStream fis = new FileInputStream(secureBootPropFile); + BufferedReader br = new BufferedReader(new InputStreamReader(fis)); + String str; + while ((str = br.readLine()) != null) { + secureBootProp = str.split(" = ")[1]; + } + } + catch (IOException io){ + Log.getLogWriter().info("Caught IO exception in startLdapServer method " + io.getMessage()); + } + Log.getLogWriter().info("The secureBootProps in getSecureBootProp method are " + secureBootProp); + return secureBootProp; } public static void HydraTask_getClientConnection() throws SQLException { @@ -78,24 +156,27 @@ public static void verifyTableData(Connection conn,String user) throws SQLExcept public static void grantSchemaPermisson(Boolean isGrant){ Vector userVector = SnappySecurityPrms.getUserName(); - String user = userVector.elementAt(0).toString(); + String query= " "; String msg = ""; Connection conn = null; - if (isGrant) { - query = "CREATE SCHEMA " + user + " AUTHORIZATION " + user; - msg = "Create scheam query is "; - } - Log.getLogWriter().info(msg + query); - try { - conn = getSecuredLocatorConnection("user1", "user123"); - conn.createStatement().execute(query); - } catch (SQLException e) { - Log.getLogWriter().info(" Caught Exception " + e.getMessage()); + for(int i=0 ;i tempPolicyMap = policyUserMap; + for (Map.Entry itrr : tempPolicyMap.entrySet()) { + String policyNm = itrr.getKey(); + String schemaOwner = itrr.getValue(); + try { + conn = getSecuredLocatorConnection(schemaOwner, schemaOwner); + } catch (Exception e) { + Log.getLogWriter().info("The exception is " + e.getMessage()); + } + Log.getLogWriter().info("Dropping policy p " + policyNm + " with schemaOwner " + schemaOwner); + conn.createStatement().execute("DROP POLICY " + policyNm); + } + } catch (Exception ex) { + throw new TestException("Caught Exception in executing the dropPolicy()" + ex.getMessage()); + } + Log.getLogWriter().info("dropPolicy() method finished successfully "); + } + + public static void createPolicy() { + Log.getLogWriter().info("Inside createPolicy() "); + Vector userVector = SnappySecurityPrms.getUserName(); + Vector onSchema = SnappySecurityPrms.getSchema(); + Vector dmlOps = SnappySecurityPrms.getDmlOps(); + int policyCnt = SnappySecurityPrms.getPolicyCnt(); + String filterCond, orderBy; + Connection conn; + try { + //create policy p2 on salary1 for select to user1 using name='a1'; + int cnt = 1; + for (int i = 0; i < userVector.size(); i++) { + String policyUser = userVector.elementAt(i).toString(); + for (int p = 0; p < policyCnt; p++) { + for (int s = 0; s < onSchema.size(); s++) { + String schemaOwnerTab = (String)onSchema.elementAt(s); + Log.getLogWriter().info("The schemaOwnerTab is " + schemaOwnerTab + " and size is " + onSchema.size()); + String[] schemaOwner = schemaOwnerTab.split("\\."); + Log.getLogWriter().info("The schemaOwner is " + schemaOwner[0]); + if (schemaOwner[1].equalsIgnoreCase("employees")) { + filterCond = "EMPLOYEEID = 1 AND COUNTRY = 'USA'"; + orderBy = " ORDER BY EMPLOYEEID asc "; + } else { + filterCond = "CATEGORYID = 4"; + orderBy = " ORDER BY CATEGORYID asc "; + } + + String policyName = "p" + p; + while (policyUserMap.containsKey(policyName)) { + Log.getLogWriter().info("checking for policy name " + policyName); + policyName = "p" + cnt; + Log.getLogWriter().info("The next policy name is " + policyName); + cnt++; + } + String policyStr = "CREATE POLICY " + policyName + " ON " + schemaOwnerTab + " FOR " + dmlOps.elementAt(0) + " TO " + policyUser + " USING " + filterCond; + //Equivalent select query will be : + String selectQryWF = "SELECT * FROM " + schemaOwnerTab + " WHERE " + filterCond; + String selectQry = "SELECT * FROM " + schemaOwnerTab + orderBy; + Map queryResultMap = new HashMap<>(); + Map queryResultMap1 = new HashMap<>(); + Log.getLogWriter().info("Policy created for " + policyUser + " on table " + onSchema.elementAt(s) + " is " + policyStr); + try { + conn = getSecuredLocatorConnection(schemaOwner[0], schemaOwner[0]); + conn.createStatement().execute(policyStr); + ResultSet rs = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY).executeQuery(selectQryWF);//select query with filter conditions + ResultSet rs1 = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY).executeQuery(selectQry);//select query with no filter conditions + //Hash map consisting of ,user + queryResultMap.put(selectQryWF, rs); + queryResultMap1.put(selectQry, rs1); + policySelectQueryMap.put(queryResultMap, policyUser); + policyUserMap.put(policyName, schemaOwner[0]); + policyFullSelectQueryMap.put(queryResultMap1, policyUser); + } catch (SQLException e) { + throw new TestException("Caught Exception in executing the policy sql" + e.getMessage() + " stackTrace = " + e.getCause()); + } + } + } + } + } catch (Exception ex) { + throw new TestException("Caught Exception in createPolicy() " + ex.getMessage()); + } + Log.getLogWriter().info("createPolicy() method finished successfully "); + } + +public static void validateQuery() { + Log.getLogWriter().info("Inside validateQuery() "); + Connection conn = null; + Boolean isDropPolicy = SnappySecurityPrms.getIsDropPolicy(); + Boolean isAltTableRLS = SnappySecurityPrms.getIsAltTableRLS(); + HashMap, String> queryUserMap; + if (isDropPolicy || !isAltTableRLS) { + queryUserMap = policyFullSelectQueryMap; // this map contains rs fro select * from table query + Log.getLogWriter().info("Using policyFullSelectQueryMap"); + } else { + queryUserMap = policySelectQueryMap; // this map contains rs fro select * from table with condition query + Log.getLogWriter().info("Using policySelectQueryMap"); + } + ResultSetMetaData currRSMD; + ResultSetMetaData prevRSMD; + ResultSet currRS; + ResultSet currFiltrRS; + try { + for (Map.Entry, String> entry : queryUserMap.entrySet()) { + Map queryMap = entry.getKey(); + String policyUser = entry.getValue(); + Log.getLogWriter().info("The user is " + policyUser); + for (Map.Entry itrr : queryMap.entrySet()) { + String selectQry = itrr.getKey(); + String[] finalSelectQ = selectQry.split("WHERE"); + ResultSet prevRS = itrr.getValue(); + Log.getLogWriter().info("The select Query is " + selectQry); + Log.getLogWriter().info("The Final select Query tobe executed by policy User is " + finalSelectQ[0]); + conn = getSecuredLocatorConnection(policyUser, policyUser); + try { + currRS = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY).executeQuery(finalSelectQ[0]); + currFiltrRS = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY).executeQuery(selectQry); + int currRSSize = getRSCount(currRS); + int prevRSSize = getRSCount(prevRS); + int currFiltrRSSize = getRSCount(currFiltrRS); + Log.getLogWriter().info("The currRSSize = " + currRSSize + " and prevRSSize = " + prevRSSize + " and currFiltrRSSize = " + currFiltrRSSize); + //Compare the prevRS and rs ,it should be same. + if ((currRSSize == prevRSSize) && (currFiltrRSSize == prevRSSize)) { + while (currRS.next() && prevRS.next()) { + currRSMD = currRS.getMetaData(); + prevRSMD = prevRS.getMetaData(); + int currRSColCnt = currRSMD.getColumnCount(); + int prevRSColCnt = prevRSMD.getColumnCount(); + Log.getLogWriter().info("The currRSColCnt count is " + currRSColCnt + " \n The prevRSColCnt col cnt is " + prevRSColCnt); + if (currRSColCnt == prevRSColCnt) { + for (int i = 1; i <= currRSColCnt; i++) { + Object currRowVal = currRS.getObject(i); + Object prevRowVal = prevRS.getObject(i); + if (currRowVal != null && prevRowVal != null) + Log.getLogWriter().info(" CLass of object is = " + currRowVal.getClass() + " \n Converted to string is " + currRowVal.toString() + " \n Policy user row val = " + prevRowVal.toString()); + //Compare the row vals it should be same. + } + } else { + Log.getLogWriter().info("ERROR : The col cnts donot match "); + } + } + prevRS.beforeFirst(); + } else + Log.getLogWriter().info("Caught exception : currRSCnt = " + currRSSize + " is not equal to prevRSSize = " + prevRSSize + " and not equal to currFiltrRSSize = " + currFiltrRSSize); + } catch (SQLException ex) { + Log.getLogWriter().info("Got sql exception " + ex.getMessage()); + } + } + } + } catch (Exception ex) { + throw new TestException("Caught Exception in validateQuery " + ex.getMessage()); + } + Log.getLogWriter().info("validateQuery() method finished successfully "); + } + + + public static int getRSCount(ResultSet resultSet) { + int size = 0; + try { + resultSet.last(); + size = resultSet.getRow(); + resultSet.beforeFirst(); + } catch (Exception ex) { + Log.getLogWriter().info("Caught exception " + ex.getMessage()); + return 0; + } + return size; + } + + public static void HydraTask_createPolicy() { + createPolicy(); + } + + public static void HydraTask_dropPolicy() { + dropPolicy(); + } + public static ArrayList getQueryArr(String fileName, String user) { Log.getLogWriter().info("Inide getQueryArray"); Log.getLogWriter().info("User = " + user); @@ -180,7 +478,7 @@ public static ArrayList getQueryArr(String fileName, String user) { Log.getLogWriter().info("splitData length is " + splitData.length); for (int i = 0; i < splitData.length; i++) { if (!(splitData[i] == null) || !(splitData[i].length() == 0)) { - String qry = splitData[i].replace("user2", schemaOwner); + String qry = splitData[i].replace("gemfire2", schemaOwner); queries.add(qry); } } @@ -196,27 +494,27 @@ public static void switchCase(String user, String passwd) throws SQLException { Boolean isGrant = SnappySecurityPrms.getIsGrant(); Boolean isPublic = SnappySecurityPrms.getIsPublic(); switch (user) { - case "user1": + case "gemfire1": runQuery(user, passwd, true); break; - case "user2": + case "gemfire2": runQuery(user, passwd, true); break; - case "user3": + case "gemfire3": if (isGrant) isAuthorized = true; else isAuthorized = false; runQuery(user, passwd, isAuthorized); break; - case "user4": + case "gemfire4": if (isGrant) isAuthorized = true; else isAuthorized = false; runQuery(user, passwd, isAuthorized); break; - case "user5" : + case "gemfire5" : if(isGrant && isPublic) isAuthorized = true; else @@ -379,7 +677,7 @@ private static Connection getConnection(String protocol, String driver, Properti throws SQLException { Log.getLogWriter().info("Creating secure connection using " + driver + " with " + protocol + - " and credentials = " + props.getProperty("user") + props.getProperty("password")); + " and credentials = " + props.getProperty("user") + " " + props.getProperty("password")); loadDriver(driver); Connection conn = DriverManager.getConnection(protocol, props); return conn; diff --git a/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnGrantRevokeOpsWithJdbcClientTest.conf b/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnGrantRevokeOpsWithJdbcClientTest.conf index 0f6499ca3e..2dd26f2ba4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnGrantRevokeOpsWithJdbcClientTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnGrantRevokeOpsWithJdbcClientTest.conf @@ -1,8 +1,8 @@ hydra.Prms-testRequirement = "Test to verify authentication of custer startUp and authorization of DMLs with Security flag on with JDBC client"; hydra.Prms-testDescription = "In this test a secure snappy cluster is started by admin user. - Then user2 creates column and row tables through snapy-shell and - grants permission to to dml operations to user3 and user4. + Then gemfire2 creates column and row tables through snapy-shell and + grants permission to to dml operations to gemfire3 and gemfire4. All the available users authorized/unauthorized perform dml operations. Then admin revokes the dml opertion permissions and again users perform dml operations.In both the cases validation is checked by comparing the number @@ -11,32 +11,74 @@ hydra.Prms-testDescription = "In this test a secure snappy cluster is started by INCLUDE $JTESTS/io/snappydata/hydra/security/startSnappyCluster.conf; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 threadGroups = snappyThreads; +//Check authorization for external tables with grant +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isExtTblSecurity = true + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocationExtTbl} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 20 + io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + +//Check authorization for external tables with revoke +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isExtTblSecurity = true + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isRevoke = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocationExtTbl} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE + io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 30 + io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = false + threadGroups = snappyThreads; + //Check dml ops authorization with GRANT for individual users INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 44 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -45,18 +87,18 @@ INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMet //Check dml ops authorization with REVOKE for individual users INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 io.snappydata.hydra.security.SnappySecurityPrms-isRevoke = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 66 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -65,19 +107,19 @@ INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMet //Check dml ops authorization with GRANT to PUBLIC INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true io.snappydata.hydra.security.SnappySecurityPrms-isPublicAccess = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 33 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -86,19 +128,19 @@ INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMet //Check dml ops authorization with REVOKE to PUBLIC INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-isRevoke = true io.snappydata.hydra.security.SnappySecurityPrms-isPublicAccess = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 66 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -107,18 +149,18 @@ INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMet //Check dml ops authorization with GRANT to only selected ops for individual users INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 62 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -139,6 +181,8 @@ sql.SQLPrms-useGfxdConfig = true; io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; -io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -DCHECK_EXTERNAL_TABLE_AUTHZ=true "; + diff --git a/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnOpsAuthzWithJdbcClientTest.conf b/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnOpsAuthzWithJdbcClientTest.conf index dde21c6fb0..384e5bebff 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnOpsAuthzWithJdbcClientTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/security/clusterAuthnOpsAuthzWithJdbcClientTest.conf @@ -6,23 +6,23 @@ hydra.Prms-testDescription = "Test to verify secure cluster startUp and uthoriza INCLUDE $JTESTS/io/snappydata/hydra/security/startSnappyCluster.conf; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 66 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -33,7 +33,10 @@ INCLUDE $JTESTS/io/snappydata/hydra/security/stopSnappyCluster.conf; io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; -io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; + + diff --git a/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthnWithServerHA.conf b/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthnWithServerHA.conf index 7de2ebe5d9..c628df100f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthnWithServerHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthnWithServerHA.conf @@ -36,19 +36,19 @@ THREADGROUP snappyThreads clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${B}\", ${${B}Hosts}, true)" ncf; -THREADGROUP user1Threads +THREADGROUP gemfire1Threads totalThreads = 2 totalVMs = 1 clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${D}\", ${${D}Hosts}, true)" ncf; -THREADGROUP user2Threads +THREADGROUP gemfire2Threads totalThreads = 2 totalVMs = 1 clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${E}\", ${${E}Hosts}, true)" ncf; -THREADGROUP user3Threads +THREADGROUP gemfire3Threads totalThreads = 2 totalVMs = 1 clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${F}\", @@ -98,61 +98,61 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy threadGroups = leadThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 threadGroups = snappyThreads; INCLUDE $JTESTS/io/snappydata/hydra/northwind/serverHA.inc; INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpServerConfig.inc; TASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_getClientConnection - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 - threadGroups = user1Threads + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 + threadGroups = gemfire1Threads maxTimesToRun = 10 ; TASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_getClientConnection - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 - threadGroups = user2Threads + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + threadGroups = gemfire2Threads maxTimesToRun = 10 ; TASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_getClientConnection - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 - threadGroups = user3Threads + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 + threadGroups = gemfire3Threads maxTimesToRun = 10 ; CLOSETASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 44 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -203,10 +203,9 @@ hydra.VmPrms-extraVMArgsSUN += "-XX:PermSize=64M -XX:MaxPermSize=256m"; io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; -io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; -/*io.snappydata.hydra.cluster.SnappyPrms-numBootStrapTrials = 50;*/ -/*io.snappydata.hydra.cluster.SnappyPrms-leadMemory = 4G;*/ diff --git a/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthzWithLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthzWithLeadHA.conf index 3f04ccad28..fe10ca1d6b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthzWithLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/security/concUserAuthzWithLeadHA.conf @@ -1,9 +1,9 @@ hydra.Prms-testRequirement = "Test to verify authentication of custer startUp and authorization of DMLs with multiple users with Security flag on with JDBC client"; hydra.Prms-testDescription = "In this test a secure snappy cluster is started by admin user. - Then user2 creates column and row tables through snapy-shell and - grants permission to to dml operations to user3 and user4. Then - user3 and user4 along with other unauthorized users perform various + Then gemfire2 creates column and row tables through snapy-shell and + grants permission to to dml operations to gemfire3 and gemfire4. Then + gemfire3 and gemfire4 along with other unauthorized users perform various dml operations while Lead is going down/up simultaneously.The test is then validated by matching the number of expected and unexpected Exceptions caught."; @@ -11,23 +11,23 @@ hydra.Prms-testDescription = "In this test a secure snappy cluster is started by INCLUDE $JTESTS/io/snappydata/hydra/security/startSnappyCluster.conf; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; @@ -35,10 +35,10 @@ INCLUDE $JTESTS/io/snappydata/hydra/northwind/leadHA.inc; INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpLeadConfig.inc; TASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 44 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -47,10 +47,10 @@ TASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMet threadGroups = snappyThreads; CLOSETASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 user5 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 user5123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire3 gemfire4 gemfire5 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 44 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -71,7 +71,8 @@ sql.SQLPrms-useGfxdConfig = true; io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; -io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://localhost:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; diff --git a/dtests/src/test/java/io/snappydata/hydra/security/local.security.conf b/dtests/src/test/java/io/snappydata/hydra/security/local.security.conf new file mode 100644 index 0000000000..1afa609972 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/local.security.conf @@ -0,0 +1,2 @@ +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; + diff --git a/dtests/src/test/java/io/snappydata/hydra/security/rls/basicRLSTest.conf b/dtests/src/test/java/io/snappydata/hydra/security/rls/basicRLSTest.conf new file mode 100644 index 0000000000..c74350018c --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/rls/basicRLSTest.conf @@ -0,0 +1,101 @@ +hydra.Prms-testRequirement = "Test to verify the basic policy creation/deletion functionality with JDBC client"; +hydra.Prms-testDescription = "Test to verify the basic policy creation/deletion functionality with JDBC client"; + +INCLUDE $JTESTS/io/snappydata/hydra/security/startSnappyCluster.conf; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts + io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql + io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + threadGroups = snappyThreads; + +//GRANT select permissions +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + threadGroups = snappyThreads; + +//Create policies +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_createPolicy + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + threadGroups = snappyThreads; + +//Enable RLS +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_enableRLS + io.snappydata.hydra.security.SnappySecurityPrms-isAltTableRLS = true + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + threadGroups = snappyThreads; + +//Verify policy creation by executing SELECT +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isAltTableRLS = true + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + +//Disable RLS +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_enableRLS + io.snappydata.hydra.security.SnappySecurityPrms-isAltTableRLS = false + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + threadGroups = snappyThreads; + +//Verify +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isAltTableRLS = false + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + +//Drop policies +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_dropPolicy + io.snappydata.hydra.security.SnappySecurityPrms-isDropPolicy = true + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-isDropPolicy = true + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + + +INCLUDE $JTESTS/io/snappydata/hydra/security/stopSnappyCluster.conf; + +hydra.Prms-totalTaskTimeSec = 100; +hydra.Prms-maxResultWaitSec = 1800; + +hydra.Prms-maxCloseTaskResultWaitSec = 1800; +hydra.Prms-serialExecution = true; + +hydra.CachePrms-names = defaultCache; +sql.SQLPrms-useGfxdConfig = true; + +io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; +io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; +io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; +io.snappydata.hydra.security.SnappySecurityPrms-isRLS = true; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps += " -J-Dsnappydata.enable-rls=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps += " -J-Dsnappydata.enable-rls=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps += " -J-Dsnappydata.enable-rls=true "; diff --git a/dtests/src/test/java/io/snappydata/hydra/security/rls/basicRLSWithSnappyJob.conf b/dtests/src/test/java/io/snappydata/hydra/security/rls/basicRLSWithSnappyJob.conf new file mode 100644 index 0000000000..aab4ca86c6 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/rls/basicRLSWithSnappyJob.conf @@ -0,0 +1,47 @@ +hydra.Prms-testRequirement = "Test to verify the basic policy creation/deletion functionality with SnappyJob"; +hydra.Prms-testDescription = "Test to verify the basic policy creation/deletion functionality with SnappyJob"; + +INCLUDE $JTESTS/io/snappydata/hydra/security/startSnappyCluster.conf; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts + io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql + io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE + threadGroups = snappyThreads; + +// This will create policy,enable RLS ,validate the query. +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.security.rls.RLSSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = " schema1=gemfire2.employees --conf schema2=gemfire2.categories --conf user3=gemfire3 --conf user4=gemfire4 " + io.snappydata.hydra.security.SnappySecurityPrms-passFile = "${passFilePath}/user3Credentials.txt" + threadGroups = snappyThreads + ; +INCLUDE $JTESTS/io/snappydata/hydra/security/stopSnappyCluster.conf; + +hydra.Prms-totalTaskTimeSec = 100; +hydra.Prms-maxResultWaitSec = 1800; + +hydra.Prms-maxCloseTaskResultWaitSec = 1800; +hydra.Prms-serialExecution = true; + +hydra.CachePrms-names = defaultCache; +sql.SQLPrms-useGfxdConfig = true; + +io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; +io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; +io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; +io.snappydata.hydra.security.SnappySecurityPrms-isRLS = true; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps += " -J-Dsnappydata.enable-rls=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps += " -J-Dsnappydata.enable-rls=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps += " -J-Dsnappydata.enable-rls=true "; + diff --git a/dtests/src/test/java/io/snappydata/hydra/security/rls/rlsTestWithClusterRestart.conf b/dtests/src/test/java/io/snappydata/hydra/security/rls/rlsTestWithClusterRestart.conf new file mode 100644 index 0000000000..c2ceb809df --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/rls/rlsTestWithClusterRestart.conf @@ -0,0 +1,77 @@ +hydra.Prms-testRequirement = "Test to verify the basic policy creation/deletion functionality with JDBC client and to verify , if policy is intact even after cluster restart."; +hydra.Prms-testDescription = "Test to verify the basic policy creation/deletion functionality with JDBC client and to verify , if policy is intact even after cluster restart."; + +INCLUDE $JTESTS/io/snappydata/hydra/security/startSnappyCluster.conf; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts + io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql + io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + threadGroups = snappyThreads; + +//GRANT select permissions +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + threadGroups = snappyThreads; + +//Create policies +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_createPolicy + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + threadGroups = snappyThreads; + +//Enable RLS +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_enableRLS + io.snappydata.hydra.security.SnappySecurityPrms-isAltTableRLS = true + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + threadGroups = snappyThreads; + +//Verify policy creation by executing SELECT +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isAltTableRLS = true + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + +INCLUDE $JTESTS/io/snappydata/hydra/startUp/clusterRestart.inc; + +CLOSETASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isAltTableRLS = true + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + +INCLUDE $JTESTS/io/snappydata/hydra/security/stopSnappyCluster.conf; + +hydra.Prms-totalTaskTimeSec = 100; +hydra.Prms-maxResultWaitSec = 1800; + +hydra.Prms-maxCloseTaskResultWaitSec = 1800; +hydra.Prms-serialExecution = true; + +hydra.CachePrms-names = defaultCache; +sql.SQLPrms-useGfxdConfig = true; + +io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; +io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; +io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; +io.snappydata.hydra.security.SnappySecurityPrms-isRLS = true; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps += " -J-Dsnappydata.enable-rls=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps += " -J-Dsnappydata.enable-rls=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps += " -J-Dsnappydata.enable-rls=true "; diff --git a/dtests/src/test/java/io/snappydata/hydra/security/rls/snappyRLS.bt b/dtests/src/test/java/io/snappydata/hydra/security/rls/snappyRLS.bt new file mode 100644 index 0000000000..73f894ad25 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/rls/snappyRLS.bt @@ -0,0 +1,30 @@ +io/snappydata/hydra/security/startLdapServer.conf + A=snappyStore snappyStoreHosts=1 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + ldapScriptPath="$GEMFIRE/../../../store/ldap-test-server" + ldapLogPath="$GEMFIRE/secureBootProp.log" + +io/snappydata/hydra/security/rls/rlsTestWithClusterRestart.conf + A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=1 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=client clientHosts=1 clientVMsPerHost=1 clientThreadsPerVM=1 + dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" + dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" + +io/snappydata/hydra/security/rls/basicRLSTest.conf + A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=1 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=client clientHosts=1 clientVMsPerHost=1 clientThreadsPerVM=1 + dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" + dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" + +io/snappydata/hydra/security/rls/basicRLSWithSnappyJob.conf + A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=1 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=client clientHosts=1 clientVMsPerHost=1 clientThreadsPerVM=1 + dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" + dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" + passFilePath = "$GEMFIRE/../../../dtests/src/resources/scripts/security" + diff --git a/dtests/src/test/java/io/snappydata/hydra/security/snappySecurity.bt b/dtests/src/test/java/io/snappydata/hydra/security/snappySecurity.bt index 821fec9d43..98296c0ce8 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/snappySecurity.bt +++ b/dtests/src/test/java/io/snappydata/hydra/security/snappySecurity.bt @@ -1,3 +1,8 @@ +io/snappydata/hydra/security/startLdapServer.conf + A=snappyStore snappyStoreHosts=1 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + ldapScriptPath="$GEMFIRE/../../../store/ldap-test-server" + ldapLogPath="$GEMFIRE/secureBootProp.log" + io/snappydata/hydra/security/clusterAuthnOpsAuthzWithJdbcClientTest.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=1 @@ -6,23 +11,35 @@ io/snappydata/hydra/security/clusterAuthnOpsAuthzWithJdbcClientTest.conf dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" - io/snappydata/hydra/security/clusterAuthnGrantRevokeOpsWithJdbcClientTest.conf +io/snappydata/hydra/security/userAuthnAuthzWithSnappyJob.conf + A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=1 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=client clientHosts=1 clientVMsPerHost=1 clientThreadsPerVM=1 + dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" + dataLocation="$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" + dataLocationExtTbl = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlExtTblQuery.sql" + passFilePath="$GEMFIRE/../../../dtests/src/resources/scripts/security/" + +io/snappydata/hydra/security/clusterAuthnGrantRevokeOpsWithJdbcClientTest.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=1 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=client clientHosts=1 clientVMsPerHost=1 clientThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" + dataLocationExtTbl = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlExtTblQuery.sql" - io/snappydata/hydra/security/userAuthnAuthzWithSmartConnTest.conf - A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 - B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 - dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" - dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" +io/snappydata/hydra/security/userAuthnAuthzWithSmartConnTest.conf + A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 + dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" + dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" + dataLocationExtTbl = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlExtTblQuery.sql" - io/snappydata/hydra/security/concUserAuthzWithLeadHA.conf +io/snappydata/hydra/security/concUserAuthzWithLeadHA.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=1 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 @@ -31,29 +48,14 @@ io/snappydata/hydra/security/clusterAuthnOpsAuthzWithJdbcClientTest.conf dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" numVMsToStop=1 - io/snappydata/hydra/security/concUserAuthnWithServerHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 - B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=user1 user1Hosts=1 user1VMsPerHost=1 user1ThreadsPerVM=2 - E=user2 user2Hosts=1 user2VMsPerHost=1 user2ThreadsPerVM=2 - F=user3 user3Hosts=1 user3VMsPerHost=1 user3ThreadsPerVM=2 - dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" - dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" - numVMsToStop=1 - -/* io/snappydata/hydra/security/userAuthnAuthzWithSnappyJob.conf - A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=1 - C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=client clientHosts=1 clientVMsPerHost=1 clientThreadsPerVM=1 - dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" - queryFile="$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" - schema1="user2.employees" - schema2="user2.categories" - isGrant=true - expectedCntWithGrant=11 unExpectedCntWithGrant=0 passfile="$GEMFIRE/../../../dtests/src/resources/scripts/security/user3Credentials.txt" - -*/ - +io/snappydata/hydra/security/concUserAuthnWithServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=user1 user1Hosts=1 user1VMsPerHost=1 user1ThreadsPerVM=2 + E=user2 user2Hosts=1 user2VMsPerHost=1 user2ThreadsPerVM=2 + F=user3 user3Hosts=1 user3VMsPerHost=1 user3ThreadsPerVM=2 + dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" + dataLocation = "$GEMFIRE/../../../dtests/src/resources/scripts/security/dmlRowColQuery.sql" + numVMsToStop=1 diff --git a/dtests/src/test/java/io/snappydata/hydra/security/startDualModeCluster.conf b/dtests/src/test/java/io/snappydata/hydra/security/startDualModeCluster.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dtests/src/test/java/io/snappydata/hydra/security/startLdapServer.bt b/dtests/src/test/java/io/snappydata/hydra/security/startLdapServer.bt new file mode 100644 index 0000000000..bb47485ab0 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/startLdapServer.bt @@ -0,0 +1,3 @@ +io/snappydata/hydra/security/startLdapServer.conf + A=snappyStore snappyStoreHosts=1 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 + ldapScriptPath="$GEMFIRE/../../../store/ldap-test-server" diff --git a/dtests/src/test/java/io/snappydata/hydra/security/startLdapServer.conf b/dtests/src/test/java/io/snappydata/hydra/security/startLdapServer.conf new file mode 100644 index 0000000000..965456009e --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/startLdapServer.conf @@ -0,0 +1,24 @@ +hydra.Prms-testRequirement = "Test to start the ldapServer"; +hydra.Prms-testDescription = "This test starts the LdapServer"; + +INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; +INCLUDE $JTESTS/hydraconfig/topology_1.inc; + +hydra.GemFirePrms-names = gemfire1; +hydra.ClientPrms-gemfireNames = gemfire1; +hydra.GemFirePrms-distributedSystem = ds; + +THREADGROUP snappyStoreThreads + totalThreads = 1 + totalVMs = 1 + clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${A}\", + ${${A}Hosts}, true)" ncf; + +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_initializeSnappyTest + runMode = always + threadGroups = snappyStoreThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_startLdapServer + io.snappydata.hydra.cdcConnector.SnappyCDCPrms-dataLocation = ${ldapScriptPath} + threadGroups = snappyStoreThreads; + diff --git a/dtests/src/test/java/io/snappydata/hydra/security/startSnappyCluster.conf b/dtests/src/test/java/io/snappydata/hydra/security/startSnappyCluster.conf index 6dacd28a0f..bce908564b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/startSnappyCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/security/startSnappyCluster.conf @@ -3,6 +3,7 @@ hydra.Prms-testDescription = "This test starts the snappy embedded as well as sp INCLUDE $JTESTS/hydraconfig/hydraparams1.inc; INCLUDE $JTESTS/hydraconfig/topology_4.inc; +//INCLUDE $JTESTS/io/snappydata/hydra/cluster/commonProps.inc; hydra.GemFirePrms-names = gemfire1; hydra.ClientPrms-gemfireNames = gemfire1; @@ -26,12 +27,6 @@ THREADGROUP locatorThreads clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${C}\", ${${C}Hosts}, true)" ncf; -/*THREADGROUP workerThreads - totalThreads = fcn "(${${D}Hosts} * ${${D}VMsPerHost} * ${${D}ThreadsPerVM}) " ncf - totalVMs = fcn "(${${D}Hosts} * ${${D}VMsPerHost})" ncf - clientNames = fcn "hydra.TestConfigFcns.generateNames(\"${D}\", - ${${D}Hosts}, true)" ncf;*/ - THREADGROUP snappyThreads totalThreads = 1 totalVMs = 1 @@ -76,3 +71,6 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_recordProcessIDWithHost runMode = always threadGroups = snappyThreads, locatorThreads, snappyStoreThreads, leadThreads; + +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; + diff --git a/dtests/src/test/java/io/snappydata/hydra/security/userAuthnAuthzWithSmartConnTest.conf b/dtests/src/test/java/io/snappydata/hydra/security/userAuthnAuthzWithSmartConnTest.conf index 3def3e8814..b0842ff27e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/userAuthnAuthzWithSmartConnTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/security/userAuthnAuthzWithSmartConnTest.conf @@ -5,89 +5,125 @@ hydra.Prms-testDescription = "In this test secure snappycluster is started .A us user connects to snappy cluster in smart connector mode and performs ops , exception is thrown depending on if the user is permitted or not to perform the ops "; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 threadGroups = snappyThreads; +//Check authorization for external tables with grant INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isExtTblSecurity = true + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob + io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSparkApp + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocationExtTbl} true gemfire2.staging_employees gemfire2.staging_categories 5 0 true" + io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=gemfire4 --conf spark.snappydata.store.password=gemfire4" + threadGroups = snappyThreads + ; + +//Check authorization for column/row tables with grant +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSparkApp - io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} true user2.employees user2.categories 11 0 true" - io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=user4 --conf spark.snappydata.store.password=user4123" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} true gemfire2.employees gemfire2.categories 11 0 true" + io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=gemfire4 --conf spark.snappydata.store.password=gemfire4" threadGroups = snappyThreads ; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSparkApp - io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} true user2.employees user2.categories 11 0 true" - io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=user3 --conf spark.snappydata.store.password=user3123" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} true gemfire2.employees gemfire2.categories 11 0 true" + io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=gemfire3 --conf spark.snappydata.store.password=gemfire3" threadGroups = snappyThreads ; +//Check authorization for external tables with revoke INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isExtTblSecurity = true + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT io.snappydata.hydra.security.SnappySecurityPrms-isRevoke = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user2.employees user2.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob + io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSparkApp + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocationExtTbl} false gemfire2.staging_employees gemfire2.staging_categories 30 0 true" + io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=gemfire4 --conf spark.snappydata.store.password=gemfire4" + threadGroups = snappyThreads; + +//Check authorization for column/row tables with revoke +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isRevoke = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSparkApp - io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} false user2.employees user2.categories 22 0 true" - io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=user3 --conf spark.snappydata.store.password=user3123" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} false gemfire2.employees gemfire2.categories 22 0 true" + io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=gemfire3 --conf spark.snappydata.store.password=gemfire3" threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSparkApp - io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} false user2.employees user2.categories 22 0 true" - io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=user4 --conf spark.snappydata.store.password=user4123" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${dataLocation} false gemfire2.employees gemfire2.categories 22 0 true" + io.snappydata.hydra.cluster.SnappyPrms-sparkSubmitExtraPrms = " --conf spark.snappydata.store.user=gemfire4 --conf spark.snappydata.store.password=gemfire4" threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire6 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire6 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} - io.snappydata.hydra.security.SnappySecurityPrms-userName = user3 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user3123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire6 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire6 threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd - io.snappydata.hydra.security.SnappySecurityPrms-userName = user2 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user2123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 gemfire4 io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user3.employees user3.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire6.employees gemfire6.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery - io.snappydata.hydra.security.SnappySecurityPrms-userName = user1 user2 user3 user4 - io.snappydata.hydra.security.SnappySecurityPrms-passWord = user123 user2123 user3123 user4123 + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire6 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire6 gemfire4 io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} - io.snappydata.hydra.security.SnappySecurityPrms-onSchema = user3.employees user3.categories + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire6.employees gemfire6.categories io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 22 io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 @@ -100,8 +136,9 @@ INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf; io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; -io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://192.168.1.162:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://192.168.1.162:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; -io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -auth-provider=LDAP -J-Dsnappydata.auth-ldap-server=ldap://192.168.1.162:389/ -user=user1 -password=user123 -J-Dgemfirexd.auth-ldap-search-base=cn=sales-group,ou=sales,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-dn=cn=admin,dc=example,dc=com -J-Dgemfirexd.auth-ldap-search-pw=admin123"; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -DCHECK_EXTERNAL_TABLE_AUTHZ=true "; + io.snappydata.hydra.cluster.SnappyPrms-useSmartConnectorMode = true; diff --git a/dtests/src/test/java/io/snappydata/hydra/security/userAuthnAuthzWithSnappyJob.conf b/dtests/src/test/java/io/snappydata/hydra/security/userAuthnAuthzWithSnappyJob.conf new file mode 100644 index 0000000000..96da885093 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/security/userAuthnAuthzWithSnappyJob.conf @@ -0,0 +1,134 @@ +hydra.Prms-testRequirement = "Test to verify authentication of custer startUp and authorization of DMLs + with Security in embedded mode"; +hydra.Prms-testDescription = "In this test secure snappycluster is started .A user creates row/column + tables.Admin user grants permissions to perform various dml ops .Another + user connects to snappy cluster using snappy job and performs ops , + exception is thrown depending on if the user is permitted or not to perform the ops "; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts + io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql + io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 + threadGroups = snappyThreads; + +//Check authorization for external tables with grant +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 + io.snappydata.hydra.security.SnappySecurityPrms-isExtTblSecurity = true + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJobUsingJobScript + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = " dataFilesLocation=${dataFilesLocation} --conf queryFile=${dataLocationExtTbl} --conf schema1=gemfire2.staging_employees --conf schema2=gemfire2.staging_categories --conf isGrant=true --conf expectedCntWithGrant=5 --conf unExpectedCntWithGrant=0 " + io.snappydata.hydra.security.SnappySecurityPrms-passFile = "${passFilePath}/user3Credentials.txt" + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads; + + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJobUsingJobScript + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = " dataFilesLocation=${dataFilesLocation} --conf queryFile=${dataLocation} --conf schema1=gemfire2.employees --conf schema2=gemfire2.categories --conf isGrant=true --conf expectedCntWithGrant=11 --conf unExpectedCntWithGrant=0 " + io.snappydata.hydra.security.SnappySecurityPrms-passFile = "${passFilePath}/user3Credentials.txt" + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads; + + +//Check authorization for external tables with revoke +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 + io.snappydata.hydra.security.SnappySecurityPrms-isExtTblSecurity = true + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT + io.snappydata.hydra.security.SnappySecurityPrms-isRevoke = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.staging_employees gemfire2.staging_categories + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJobUsingJobScript + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = " dataFilesLocation=${dataFilesLocation} --conf queryFile=${dataLocationExtTbl} --conf schema1=gemfire2.staging_employees --conf schema2=gemfire2.staging_categories --conf isGrant=false --conf expectedCntWithGrant=10 --conf unExpectedCntWithGrant=0 " + io.snappydata.hydra.security.SnappySecurityPrms-passFile = "${passFilePath}/user3Credentials.txt" + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire3 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isRevoke = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire2.employees gemfire2.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJobUsingJobScript + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.security.CreateAndLoadTablesSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = " dataFilesLocation=${dataFilesLocation} --conf queryFile=${dataLocation} --conf schema1=gemfire2.employees --conf schema2=gemfire2.categories --conf isGrant=false --conf expectedCntWithGrant=22 --conf unExpectedCntWithGrant=0 " + io.snappydata.hydra.security.SnappySecurityPrms-passFile = "${passFilePath}/user3Credentials.txt" + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_GrantSchemaPermisson + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_executeSQLScripts + io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = create_and_load_columnTables_persistent.sql + io.snappydata.hydra.cluster.SnappyPrms-dataLocation = ${dataFilesLocation} + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire4 + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_performGrantRevokeCmd + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire2 gemfire6 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire2 gemfire6 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire4.employees gemfire4.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE + threadGroups = snappyThreads; + +INITTASK taskClass = io.snappydata.hydra.security.SnappySecurityTest taskMethod = HydraTask_runQuery + io.snappydata.hydra.security.SnappySecurityPrms-userName = gemfire1 gemfire2 gemfire6 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-passWord = gemfire1 gemfire2 gemfire6 gemfire4 + io.snappydata.hydra.security.SnappySecurityPrms-dataLocation = ${dataLocation} + io.snappydata.hydra.security.SnappySecurityPrms-onSchema = gemfire4.employees gemfire4.categories + io.snappydata.hydra.security.SnappySecurityPrms-dmlOperations = SELECT INSERT UPDATE DELETE + io.snappydata.hydra.security.SnappySecurityPrms-expectedExcptCnt = 22 + io.snappydata.hydra.security.SnappySecurityPrms-unExpectedExcptCnt = 0 + io.snappydata.hydra.security.SnappySecurityPrms-isGrant = true + threadGroups = snappyThreads; + + + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf; +io.snappydata.hydra.security.SnappySecurityPrms-isSecurity = true; +//io.snappydata.hydra.cluster.SnappyPrms-useSmartConnectorMode = true; +io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar; +io.snappydata.hydra.cluster.SnappyPrms-shufflePartitions = 6; +io.snappydata.hydra.cluster.SnappyPrms-serverMemory = 4g; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true "; +io.snappydata.hydra.cluster.SnappyPrms-leaderLauncherProps = " -J-Dsnappydata.RESTRICT_TABLE_CREATION=true -DCHECK_EXTERNAL_TABLE_AUTHZ=true"; + + + diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java index ed48144818..3bfb76c1b7 100644 --- a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java @@ -43,6 +43,7 @@ import hydra.RemoteTestModule; import hydra.TestConfig; import hydra.blackboard.AnyCyclicBarrier; +import io.snappydata.hydra.cluster.SnappyBB; import io.snappydata.hydra.cluster.SnappyTest; import sql.ClientDiscDBManager; import sql.SQLHelper; @@ -147,7 +148,12 @@ public static void HydraTask_initializeSelectThreads() { public static void HydraTask_initializeTablesMetaData() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); testInstance.saveTableMetaDataToBB(conn); } catch (SQLException se) { throw new TestException("Got exception while getting connection. Exception is : ", se); @@ -189,7 +195,12 @@ public static void HydraTask_performDMLOp() { public void performDMLOp() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); //perform DML operation which can be insert, update, delete. String operation = SnapshotIsolationPrms.getDMLOperations(); switch (DMLOp.getOperation(operation)) { @@ -224,7 +235,12 @@ public static void HydraTask_testMultipleSnapshot() { public void testMultipleSnapshot() { try { int iterations = 5; - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; ResultSet[] snappyRS = new ResultSet[iterations]; ResultSet derbyRS = null; @@ -307,7 +323,12 @@ public static void HydraTask_executeQueries() { public void executeQuery() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; String query = SnapshotIsolationPrms.getSelectStmts(); Log.getLogWriter().info("Blocking operations in snappy."); @@ -580,7 +601,12 @@ public static void HydraTask_performBatchInsert() { public void performBatchInsert() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; int batchSize = 100; String[] dmlTable = SnapshotIsolationPrms.getDMLTables(); @@ -700,7 +726,12 @@ public static void HydraTask_multipleExecuteQueries() { public void multipleSelectQuery() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean)SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); String query = ""; int myTid = RemoteTestModule.getCurrentThread().getThreadId(); //total number of threads executing select query @@ -801,7 +832,12 @@ public void verifyResults() { try { String[] tables = SnapshotIsolationPrms.getTableNames(); String stmt = "select * from "; - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = getDerbyConnection(); for (String table : tables) { tableName = table; @@ -963,7 +999,12 @@ public static void HydraTask_createSnappySchemas() { protected void createSnappySchemas() { try{ - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); Log.getLogWriter().info("creating schemas in snappy."); createSchemas(conn,false); Log.getLogWriter().info("done creating schemas in snappy."); @@ -1032,7 +1073,12 @@ public static synchronized void HydraTask_createSnappyTables(){ protected void createSnappyTables() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); Log.getLogWriter().info("dropping tables in snappy."); dropTables(conn); //drop table before creating it Log.getLogWriter().info("done dropping tables in snappy"); @@ -1113,7 +1159,12 @@ public static synchronized void HydraTask_populateTables(){ protected void populateTables() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1","gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; if (hasDerbyServer) dConn = getDerbyConnection(); diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/nw_schema.inc b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/nw_schema.inc index 4ce44c18d3..91eed6b550 100644 --- a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/nw_schema.inc +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/nw_schema.inc @@ -1,31 +1,31 @@ -sql.SQLPrms-createSchemas = "create schema app"; +//sql.SQLPrms-createSchemas = "create schema gemfire1"; sql.SQLPrms-createTablesStatements = -"create table app.regions (RegionID int,RegionDescription varchar(100),tid int)" -"create table app.categories (CategoryID int,CategoryName varchar(100),Description varchar(100),Picture varchar(300),tid int)" -"create table app.shippers (ShipperID int not null,CompanyName varchar(100) not null,Phone varchar(100),tid int)" -"create table app.employees(EmployeeID int,LastName varchar(100),FirstName varchar(100),Title varchar(100),TitleOfCourtesy varchar(100),BirthDate timestamp,HireDate timestamp,Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100), HomePhone varchar(100),Extension varchar(100),Photo varchar(300),Notes varchar(300),ReportsTo int,PhotoPath varchar(100),tid int)" -"create table app.customers(CustomerID varchar(100),CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),tid int)" -"create table app.orders (OrderID int,CustomerID varchar(20),EmployeeID int,OrderDate timestamp,RequiredDate timestamp,ShippedDate timestamp,ShipVia int,Freight double,ShipName varchar(50),ShipAddress varchar(50),ShipCity varchar(15), ShipRegion varchar(20),ShipPostalCode varchar(10),ShipCountry varchar(20),tid int)" -"create table app.order_details (OrderID int,ProductID int,UnitPrice double,Quantity smallint,Discount double,tid int)" -"create table app.products(ProductID int,ProductName varchar(100),SupplierID int,CategoryID int,QuantityPerUnit varchar(100),UnitPrice double,UnitsInStock smallint,UnitsOnOrder smallint,ReorderLevel smallint,Discontinued smallint,tid int)" -"create table app.suppliers(SupplierID int,CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),HomePage varchar(100),tid int) " -"create table app.territories(TerritoryID varchar(100),TerritoryDescription varchar(100),RegionID varchar(100),tid int)" -"create table app.employee_territories(EmployeeID int,TerritoryID varchar(100),tid int)" +"create table regions (RegionID int,RegionDescription varchar(100),tid int)" +"create table categories (CategoryID int,CategoryName varchar(100),Description varchar(100),Picture varchar(300),tid int)" +"create table shippers (ShipperID int not null,CompanyName varchar(100) not null,Phone varchar(100),tid int)" +"create table employees(EmployeeID int,LastName varchar(100),FirstName varchar(100),Title varchar(100),TitleOfCourtesy varchar(100),BirthDate timestamp,HireDate timestamp,Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100), HomePhone varchar(100),Extension varchar(100),Photo varchar(300),Notes varchar(300),ReportsTo int,PhotoPath varchar(100),tid int)" +"create table customers(CustomerID varchar(100),CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),tid int)" +"create table orders (OrderID int,CustomerID varchar(20),EmployeeID int,OrderDate timestamp,RequiredDate timestamp,ShippedDate timestamp,ShipVia int,Freight double,ShipName varchar(50),ShipAddress varchar(50),ShipCity varchar(15), ShipRegion varchar(20),ShipPostalCode varchar(10),ShipCountry varchar(20),tid int)" +"create table order_details (OrderID int,ProductID int,UnitPrice double,Quantity smallint,Discount double,tid int)" +"create table products(ProductID int,ProductName varchar(100),SupplierID int,CategoryID int,QuantityPerUnit varchar(100),UnitPrice double,UnitsInStock smallint,UnitsOnOrder smallint,ReorderLevel smallint,Discontinued smallint,tid int)" +"create table suppliers(SupplierID int,CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),HomePage varchar(100),tid int) " +"create table territories(TerritoryID varchar(100),TerritoryDescription varchar(100),RegionID varchar(100),tid int)" +"create table employee_territories(EmployeeID int,TerritoryID varchar(100),tid int)" ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-tablesList = -"app.regions" -"app.categories" -"app.shippers" -"app.employees" -"app.customers" -"app.orders" -"app.order_details" -"app.products" -"app.suppliers" -"app.territories" -"app.employee_territories" +"regions" +"categories" +"shippers" +"employees" +"customers" +"orders" +"order_details" +"products" +"suppliers" +"territories" +"employee_territories" ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-dataFileLocation = ${derbyDataLocation}; @@ -45,58 +45,58 @@ io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-csvFileNames = ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-selectStmts = ONEOF -"SELECT CategoryID,CategoryName,Description FROM app.Categories" -"SELECT * FROM app.Customers" -"SELECT * FROM app.Orders" -"SELECT FirstName, LastName FROM app.Employees" -"SELECT FirstName, LastName FROM app.Employees ORDER BY LastName" -"SELECT Title, FirstName, LastName FROM app.Employees ORDER BY 1,3" -"SELECT Title, FirstName, LastName FROM app.Employees ORDER BY Title ASC, LastName DESC" -"SELECT Title, FirstName, LastName FROM app.Employees WHERE Title = 'Sales Representative'" -"SELECT FirstName, LastName FROM app.Employees WHERE Title <> 'Sales Representative'" -"SELECT FirstName, LastName FROM app.Employees WHERE LastName >= 'N'" -"SELECT FirstName, LastName FROM app.Employees WHERE Region IS NULL" -"SELECT FirstName, LastName FROM app.Employees WHERE LastName >= 'N' ORDER BY LastName DESC" -"SELECT CompanyName, ContactName, Fax FROM app.Customers WHERE Fax IS NOT NULL ORDER BY CompanyName" -"SELECT TitleOfCourtesy, FirstName, LastName FROM app.Employees WHERE TitleOfCourtesy IN ('Ms.','Mrs.')" -"SELECT TitleOfCourtesy, FirstName, LastName FROM app.Employees WHERE TitleOfCourtesy LIKE 'M%'" -"SELECT OrderID, Freight, Freight * 1.1 AS FreightTotal FROM app.Orders WHERE Freight >= 500" +"SELECT CategoryID,CategoryName,Description FROM Categories" +"SELECT * FROM Customers" +"SELECT * FROM Orders" +"SELECT FirstName, LastName FROM Employees" +"SELECT FirstName, LastName FROM Employees ORDER BY LastName" +"SELECT Title, FirstName, LastName FROM Employees ORDER BY 1,3" +"SELECT Title, FirstName, LastName FROM Employees ORDER BY Title ASC, LastName DESC" +"SELECT Title, FirstName, LastName FROM Employees WHERE Title = 'Sales Representative'" +"SELECT FirstName, LastName FROM Employees WHERE Title <> 'Sales Representative'" +"SELECT FirstName, LastName FROM Employees WHERE LastName >= 'N'" +"SELECT FirstName, LastName FROM Employees WHERE Region IS NULL" +"SELECT FirstName, LastName FROM Employees WHERE LastName >= 'N' ORDER BY LastName DESC" +"SELECT CompanyName, ContactName, Fax FROM Customers WHERE Fax IS NOT NULL ORDER BY CompanyName" +"SELECT TitleOfCourtesy, FirstName, LastName FROM Employees WHERE TitleOfCourtesy IN ('Ms.','Mrs.')" +"SELECT TitleOfCourtesy, FirstName, LastName FROM Employees WHERE TitleOfCourtesy LIKE 'M%'" +"SELECT OrderID, Freight, Freight * 1.1 AS FreightTotal FROM Orders WHERE Freight >= 500" //"SELECT SUM(Quantity) AS TotalUnits FROM Order_Details WHERE ProductID=3" -"SELECT MIN(HireDate) AS FirstHireDate, MAX(HireDate) AS LastHireDate FROM app.Employees" -//"SELECT City, COUNT(EmployeeID) AS NumEmployees FROM app.Employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees" -//"SELECT COUNT(DISTINCT City) AS NumCities FROM app.Employees" +"SELECT MIN(HireDate) AS FirstHireDate, MAX(HireDate) AS LastHireDate FROM Employees" +//"SELECT City, COUNT(EmployeeID) AS NumEmployees FROM Employees WHERE Title = 'Sales Representative' GROUP BY City HAVING COUNT(EmployeeID) > 1 ORDER BY NumEmployees" +//"SELECT COUNT(DISTINCT City) AS NumCities FROM Employees" //"SELECT ProductID, AVG(UnitPrice) AS AveragePrice FROM Products GROUP BY ProductID HAVING AVG(UnitPrice) > 70 ORDER BY AveragePrice" //"SELECT COUNT(DISTINCT e.EmployeeID) AS numEmployees, COUNT(DISTINCT c.CustomerID) AS numCompanies, e.City as employeeCity, c.City as customerCity FROM Employees e JOIN Customers c ON (e.City = c.City) GROUP BY e.City, c.City ORDER BY numEmployees DESC" //"SELECT p.ProductName, SUM(od.Quantity) AS TotalUnits FROM Order_Details od JOIN Products p ON (p.ProductID = od.ProductID) GROUP BY p.ProductName HAVING SUM(Quantity) < 200" //"SELECT ShipCountry, Sum(Order_Details.UnitPrice * Quantity * Discount) AS ProductSales FROM Orders INNER JOIN Order_Details ON Orders.OrderID = Order_Details.OrderID GROUP BY ShipCountry" -"SELECT CompanyName FROM app.Customers WHERE CustomerID = (SELECT CustomerID FROM app.Orders WHERE OrderID = 10290)" -"SELECT ProductName, SupplierID FROM app.Products WHERE SupplierID IN (SELECT SupplierID FROM app.Suppliers WHERE CompanyName IN('Exotic Liquids', 'Grandma Kellys Homestead', 'Tokyo Traders'))" -"SELECT ProductName FROM app.Products WHERE CategoryID = (SELECT CategoryID FROM app.Categories WHERE CategoryName = 'Seafood')" -"SELECT CompanyName FROM app.Suppliers WHERE SupplierID IN (SELECT SupplierID FROM app.Products WHERE CategoryID = 8)" +"SELECT CompanyName FROM Customers WHERE CustomerID = (SELECT CustomerID FROM Orders WHERE OrderID = 10290)" +"SELECT ProductName, SupplierID FROM Products WHERE SupplierID IN (SELECT SupplierID FROM Suppliers WHERE CompanyName IN('Exotic Liquids', 'Grandma Kellys Homestead', 'Tokyo Traders'))" +"SELECT ProductName FROM Products WHERE CategoryID = (SELECT CategoryID FROM Categories WHERE CategoryName = 'Seafood')" +"SELECT CompanyName FROM Suppliers WHERE SupplierID IN (SELECT SupplierID FROM Products WHERE CategoryID = 8)" //"SELECT CompanyName FROM Suppliers WHERE SupplierID IN (SELECT SupplierID FROM Products WHERE CategoryID = (SELECT CategoryID FROM Categories WHERE CategoryName = 'Seafood'))" -"SELECT e.EmployeeID, e.FirstName, e.LastName, o.OrderID, o.OrderDate FROM app.Employees e JOIN app.Orders o ON (e.EmployeeID = o.EmployeeID) ORDER BY o.OrderDate" -"SELECT e.FirstName, e.LastName, o.OrderID FROM app.Employees e JOIN app.Orders o ON (e.EmployeeID = o.EmployeeID) WHERE o.RequiredDate < o.ShippedDate ORDER BY e.LastName, e.FirstName" -"select s.supplierid,s.companyname,p.productid,p.productname from app.suppliers s join app.products p on(s.supplierid= p.supplierid) and s.companyname IN('Grandma Kellys Homestead','Tokyo Traders','Exotic Liquids')" -"SELECT c.customerID, o.orderID FROM app.customers c INNER JOIN app.orders o ON c.CustomerID = o.CustomerID" -"SELECT OrderDate, ShippedDate, CustomerID, Freight FROM app.Orders WHERE OrderDate = '1997-05-19 00:00:00.000'" -"SELECT FirstName, LastName, BirthDate FROM app.Employees WHERE BirthDate BETWEEN '1950-01-01 00:00:00.000' AND '1959-12-31 23:59:59'" -"SELECT CompanyName FROM app.Customers WHERE CustomerID IN (SELECT CustomerID FROM app.Orders WHERE OrderDate BETWEEN '1997-01-01 00:00:00.000' AND '1997-12-31 00:00:00.000')" -"SELECT o.OrderID, c.CompanyName, e.FirstName, e.LastName FROM app.Orders o JOIN app.Employees e ON (e.EmployeeID = o.EmployeeID) JOIN app.Customers c ON (c.CustomerID = o.CustomerID) WHERE o.ShippedDate > o.RequiredDate AND o.OrderDate > '1998-01-01 00:00:00.000' ORDER BY c.CompanyName" -//"select distinct (a.ShippedDate) as ShippedDate, a.OrderID, b.Subtotal, year(a.ShippedDate) as Year_ from app.Orders a inner join ( select distinct OrderID, sum(UnitPrice * Quantity * (1 - Discount)) as Subtotal from order_details group by OrderID ) b on a.OrderID = b.OrderID where a.ShippedDate is not null and a.ShippedDate > '1996-12-24 00:00:00.000' and a.ShippedDate < '1997-09-30 00:00:00.000' order by ShippedDate" -//"select distinct a.CategoryID, a.CategoryName, b.ProductName, sum(c.ExtendedPrice) as ProductSales from app.Categories a inner join Products b on a.CategoryID = b.CategoryID inner join ( select distinct y.OrderID, y.ProductID, x.ProductName, y.UnitPrice, y.Quantity, y.Discount, round(y.UnitPrice * y.Quantity * (1 - y.Discount), 2) as ExtendedPrice from Products x inner join Order_Details y on x.ProductID = y.ProductID order by y.OrderID ) c on c.ProductID = b.ProductID inner join Orders d on d.OrderID = c.OrderID where d.OrderDate > '1997-01-01 00:00:00.000' and d.OrderDate < '1997-12-31 00:00:00.000' group by a.CategoryID, a.CategoryName, b.ProductName order by a.CategoryName, b.ProductName, ProductSales" +"SELECT e.EmployeeID, e.FirstName, e.LastName, o.OrderID, o.OrderDate FROM Employees e JOIN Orders o ON (e.EmployeeID = o.EmployeeID) ORDER BY o.OrderDate" +"SELECT e.FirstName, e.LastName, o.OrderID FROM Employees e JOIN Orders o ON (e.EmployeeID = o.EmployeeID) WHERE o.RequiredDate < o.ShippedDate ORDER BY e.LastName, e.FirstName" +"select s.supplierid,s.companyname,p.productid,p.productname from suppliers s join products p on(s.supplierid= p.supplierid) and s.companyname IN('Grandma Kellys Homestead','Tokyo Traders','Exotic Liquids')" +"SELECT c.customerID, o.orderID FROM customers c INNER JOIN orders o ON c.CustomerID = o.CustomerID" +"SELECT OrderDate, ShippedDate, CustomerID, Freight FROM Orders WHERE OrderDate = '1997-05-19 00:00:00.000'" +"SELECT FirstName, LastName, BirthDate FROM Employees WHERE BirthDate BETWEEN '1950-01-01 00:00:00.000' AND '1959-12-31 23:59:59'" +"SELECT CompanyName FROM Customers WHERE CustomerID IN (SELECT CustomerID FROM Orders WHERE OrderDate BETWEEN '1997-01-01 00:00:00.000' AND '1997-12-31 00:00:00.000')" +"SELECT o.OrderID, c.CompanyName, e.FirstName, e.LastName FROM Orders o JOIN Employees e ON (e.EmployeeID = o.EmployeeID) JOIN Customers c ON (c.CustomerID = o.CustomerID) WHERE o.ShippedDate > o.RequiredDate AND o.OrderDate > '1998-01-01 00:00:00.000' ORDER BY c.CompanyName" +//"select distinct (a.ShippedDate) as ShippedDate, a.OrderID, b.Subtotal, year(a.ShippedDate) as Year_ from Orders a inner join ( select distinct OrderID, sum(UnitPrice * Quantity * (1 - Discount)) as Subtotal from order_details group by OrderID ) b on a.OrderID = b.OrderID where a.ShippedDate is not null and a.ShippedDate > '1996-12-24 00:00:00.000' and a.ShippedDate < '1997-09-30 00:00:00.000' order by ShippedDate" +//"select distinct a.CategoryID, a.CategoryName, b.ProductName, sum(c.ExtendedPrice) as ProductSales from Categories a inner join Products b on a.CategoryID = b.CategoryID inner join ( select distinct y.OrderID, y.ProductID, x.ProductName, y.UnitPrice, y.Quantity, y.Discount, round(y.UnitPrice * y.Quantity * (1 - y.Discount), 2) as ExtendedPrice from Products x inner join Order_Details y on x.ProductID = y.ProductID order by y.OrderID ) c on c.ProductID = b.ProductID inner join Orders d on d.OrderID = c.OrderID where d.OrderDate > '1997-01-01 00:00:00.000' and d.OrderDate < '1997-12-31 00:00:00.000' group by a.CategoryID, a.CategoryName, b.ProductName order by a.CategoryName, b.ProductName, ProductSales" //"SELECT FirstName || ' ' || LastName FROM Employees" -//"SELECT * FROM app.orders LEFT SEMI JOIN order_details ON orders.OrderID = order_details.OrderId" -//"SELECT * FROM app.orders LEFT SEMI JOIN order_details" +//"SELECT * FROM orders LEFT SEMI JOIN order_details ON orders.OrderID = order_details.OrderId" +//"SELECT * FROM orders LEFT SEMI JOIN order_details" //"select c.CategoryName as Product_Category, case when s.Country in ('UK','Spain','Sweden','Germany','Norway', 'Denmark','Netherlands','Finland','Italy','France') then 'Europe' when s.Country in ('USA','Canada','Brazil') then 'America' else 'Asia-Pacific' end as Supplier_Continent, sum(p.UnitsInStock) as UnitsInStock from Suppliers s inner join Products p on p.SupplierID=s.SupplierID inner join Categories c on c.CategoryID=p.CategoryID group by c.CategoryName, case when s.Country in ('UK','Spain','Sweden','Germany','Norway', 'Denmark','Netherlands','Finland','Italy','France') then 'Europe' when s.Country in ('USA','Canada','Brazil') then 'America' else 'Asia-Pacific' end" //"select CategoryName, format_number(sum(ProductSales), 2) as CategorySales from ( select distinct a.CategoryName, b.ProductName, format_number(sum(c.UnitPrice * c.Quantity * (1 - c.Discount)), 2) as ProductSales, concat('Qtr ', quarter(d.ShippedDate)) as ShippedQuarter from Categories as a inner join Products as b on a.CategoryID = b.CategoryID inner join Order_Details as c on b.ProductID = c.ProductID inner join Orders as d on d.OrderID = c.OrderID where d.ShippedDate > '1997-01-01 00:00:00.000' and d.ShippedDate < '1997-12-31 00:00:00.000' group by a.CategoryName, b.ProductName, concat('Qtr ', quarter(d.ShippedDate)) order by a.CategoryName, b.ProductName, ShippedQuarter ) as x group by CategoryName order by CategoryName" FOENO ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-dmlTables = -"app.employees" -"app.customers" -"app.orders" -"app.order_details" +"employees" +"customers" +"orders" +"order_details" ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-csvLocationforLargeData = "/export/shared/QA_DATA/northWind_1G"; @@ -109,28 +109,28 @@ io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-insertCsvFileNames = ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-insertStmts = -"insert into app.employees values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.customers values (?,?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.orders values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.order_details values (?,?,?,?,?,?)" +"insert into employees values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +"insert into customers values (?,?,?,?,?,?,?,?,?,?,?,?)" +"insert into orders values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +"insert into order_details values (?,?,?,?,?,?)" ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-updateStmts = ONEOF -"UPDATE app.products SET UnitPrice = UnitPrice * 1.1 WHERE CategoryID = 2" -"UPDATE app.products SET UnitPrice = UnitPrice * 0.9" -"UPDATE app.products SET UnitPrice = UnitPrice * 1.1, ReorderLevel = ReorderLevel + 5 WHERE CategoryID = 1" -"UPDATE app.order_details SET UnitPrice = UnitPrice * 1.1" -"UPDATE app.order_details SET UnitPrice = UnitPrice * 1.1 WHERE ORDERID < 10450" -"UPDATE app.employees SET TitleOfCourtesy='Mr.' WHERE FIRSTNAME LIKE '%31'" -"UPDATE app.orders SET ShipVia = ShipVia + 1 WHERE OrderID = 10290" -"UPDATE app.orders SET Freight = Freight * 1.1 WHERE Freight >= 500" +"UPDATE products SET UnitPrice = UnitPrice * 1.1 WHERE CategoryID = 2" +"UPDATE products SET UnitPrice = UnitPrice * 0.9" +"UPDATE products SET UnitPrice = UnitPrice * 1.1, ReorderLevel = ReorderLevel + 5 WHERE CategoryID = 1" +"UPDATE order_details SET UnitPrice = UnitPrice * 1.1" +"UPDATE order_details SET UnitPrice = UnitPrice * 1.1 WHERE ORDERID < 10450" +"UPDATE employees SET TitleOfCourtesy='Mr.' WHERE FIRSTNAME LIKE '%31'" +"UPDATE orders SET ShipVia = ShipVia + 1 WHERE OrderID = 10290" +"UPDATE orders SET Freight = Freight * 1.1 WHERE Freight >= 500" FOENO ; io.snappydata.hydra.snapshotIsolation.SnapshotIsolationPrms-deleteStmts = ONEOF -"delete from app.employees where EmployeeID = 6" -"delete from app.employees where EmployeeID < 5" -"delete from app.employees where FIRSTNAME LIKE '%2'" +"delete from employees where EmployeeID = 6" +"delete from employees where EmployeeID < 5" +"delete from employees where FIRSTNAME LIKE '%2'" FOENO ; diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java index 2e1cc24520..db0a357baa 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java @@ -191,7 +191,12 @@ public static void HydraTask_saveTableMetaDataToBB() { public void saveTableMetaDataToBB() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); String[] tableNames = SnappySchemaPrms.getTableNames(); for (String table : tableNames) { ResultSet rs = conn.createStatement().executeQuery("select * from " + table); @@ -223,7 +228,12 @@ public static void HydraTask_createSnappySchemas() { protected void createSnappySchemas() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); Log.getLogWriter().info("creating schemas in snappy."); createSchemas(conn, false); Log.getLogWriter().info("done creating schemas in snappy."); @@ -295,7 +305,12 @@ public static synchronized void HydraTask_createSnappyTables() { protected void createSnappyTables() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); Log.getLogWriter().info("dropping tables in snappy."); dropTables(conn); //drop table before creating it Log.getLogWriter().info("done dropping tables in snappy"); @@ -434,8 +449,13 @@ public void loadTablesInSnappy(String dataLocation) { int tid = getMyTid(); dynamicAppProps.put(tid, "dataFilesLocation=" + dataLocation); String logFile = "snappyJobResult_thr_" + tid + "_" + System.currentTimeMillis() + ".log"; - executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), - jarPath, SnappyPrms.getUserAppName()); + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(!isSecurityEnabled) + executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), + jarPath, SnappyPrms.getUserAppName()); + else + executeSnappyJobUsingJobScript(SnappyPrms.getSnappyJobClassNames(), SnappyPrms.getUserAppName(), logFile); + } public void loadTablesInDerby(String dataLocation) { @@ -451,8 +471,8 @@ public void loadTablesInDerby(String dataLocation) { try { PreparedStatement ps = dConn.prepareStatement( "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(?,?,?,null,null,null,0)"); - ps.setString(1, table[0]); - ps.setString(2, table[1]); + ps.setString(1, "APP"); + ps.setString(2, table[0]); ps.setString(3, csvFilePath); //ps.setString(4, ","); ps.execute(); @@ -529,7 +549,12 @@ public void run() { try { Log.getLogWriter().info("Loading data into " + tableName + " using CSV : " + csvFilePath); - Connection conn = getLocatorConnection(), dConn = null; + Connection conn = null, dConn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); if (hasDerbyServer) dConn = derbyTestUtils.getDerbyConnection(); PreparedStatement snappyPS = null, derbyPS = null; @@ -891,7 +916,12 @@ public void performInsertUsingBatch(Connection conn, String tableName, public void performInsert() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; String[] dmlTable = SnappySchemaPrms.getDMLTables(); int rand = new Random().nextInt(dmlTable.length); @@ -949,7 +979,12 @@ public void getAndExecuteSelect(Connection conn, String stmt, boolean isDerby) { public void performUpdate() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; //get the derby connection here String updateStmt[] = SnappySchemaPrms.getUpdateStmts(); int numRows = 0; @@ -999,7 +1034,12 @@ public void performUpdate() { public void performDelete() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; //get the derby connection here String deleteStmt[] = SnappySchemaPrms.getDeleteStmts(); int numRows = 0; @@ -1054,7 +1094,12 @@ public static void HydraTask_executeQueries() { public void executeQuery() { try { - Connection conn = getLocatorConnection(); + Connection conn = null; + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); Connection dConn = null; String selectStmt[] = SnappySchemaPrms.getSelectStmts(); ResultSet snappyRS, derbyRS = null; @@ -1249,7 +1294,11 @@ public String verifyResultsForTable(String selectStmt, String table, String orde StringBuffer mismatchString = new StringBuffer(); Connection conn, dConn; try { - conn = getLocatorConnection(); + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(isSecurityEnabled) + conn = getSecuredLocatorConnection("gemfire1", "gemfire1"); + else + conn = getLocatorConnection(); dConn = derbyTestUtils.getDerbyConnection(); if (useTid) selectStmt = addTidToQuery(selectStmt, getMyTid()); if (orderByClause.length() > 0) @@ -1590,8 +1639,12 @@ public void performUpdateInSnappy(ConnType connType) { if (connType.equals(ConnType.SNAPPY)) { dynamicAppProps.put(tid, "stmt=\\\"" + stmt + "\\\",tableName=" + tableName + ",tid=" + tid); String logFile = "snappyJobResult_thr_" + tid + "_" + System.currentTimeMillis() + ".log"; - executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(!isSecurityEnabled) + executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), jarPath, SnappyPrms.getUserAppName()); + else + executeSnappyJobUsingJobScript(SnappyPrms.getSnappyJobClassNames(), SnappyPrms.getUserAppName(), logFile); } else { // thin client smart connector mode dynamicAppProps.put(tid, "\"" + stmt + "\"" + " " + tid); String logFile = "sparkAppResult_thr_" + tid + "_" + System.currentTimeMillis() + ".log"; @@ -1633,8 +1686,13 @@ public void performDeleteInSnappy(ConnType connType) { if (connType.equals(ConnType.SNAPPY)) { dynamicAppProps.put(tid, "stmt=\\\"" + stmt + "\\\",tableName=" + tableName + ",tid=" + tid); String logFile = "snappyJobResult_thr_" + tid + "_" + System.currentTimeMillis() + ".log"; - executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), - jarPath, SnappyPrms.getUserAppName()); + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(!isSecurityEnabled) + executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), + jarPath, SnappyPrms.getUserAppName()); + else + executeSnappyJobUsingJobScript(SnappyPrms.getSnappyJobClassNames(), SnappyPrms.getUserAppName(), logFile); + } else { // thin client smart connector mode dynamicAppProps.put(tid, "\"" + stmt + "\"" + " " + tid); String logFile = "sparkAppResult_thr_" + tid + "_" + System.currentTimeMillis() + ".log"; @@ -1678,8 +1736,13 @@ public void performInsertInSnappy(ConnType connType) { dynamicAppProps.put(tid, "stmt=\\\"" + insertStmt + "\\\",tableName=" + tableName + "," + "tid=" + tid); String logFile = "snappyJobResult_thr_" + tid + "_" + System.currentTimeMillis() + ".log"; - executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), - jarPath, SnappyPrms.getUserAppName()); + boolean isSecurityEnabled = (Boolean) SnappyBB.getBB().getSharedMap().get("SECURITY_ENABLED"); + if(!isSecurityEnabled) + executeSnappyJob(SnappyPrms.getSnappyJobClassNames(), logFile, SnappyPrms.getUserAppJar(), + jarPath, SnappyPrms.getUserAppName()); + else + executeSnappyJobUsingJobScript(SnappyPrms.getSnappyJobClassNames(), SnappyPrms.getUserAppName(), logFile); + } else { // thin client smart connector mode dynamicAppProps.put(tid, "\"" + insertStmt + "\"" + " " + tid); String logFile = "sparkAppResult_thr_" + tid + "_" + System.currentTimeMillis() + ".log"; diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/columnTableExtn.inc b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/columnTableExtn.inc index 651c1f60cc..75ac6e1aea 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/columnTableExtn.inc +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/columnTableExtn.inc @@ -10,4 +10,4 @@ io.snappydata.hydra.testDMLOps.SnappySchemaPrms-snappyDDLExtn = " USING column OPTIONS(PARTITION_BY 'SupplierID', buckets '37',redundancy '${redundantCopies}', PERSISTENT '${persistenceMode}')" " USING column OPTIONS(partition_by 'TerritoryID', buckets '3', redundancy '${redundantCopies}', PERSISTENT '${persistenceMode}')" " USING row OPTIONS(partition_by 'EmployeeID', buckets '1', redundancy '${redundantCopies}', PERSISTENT '${persistenceMode}')" -; \ No newline at end of file +; diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/embeddedMode/testDMLOpsInSnappy.bt b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/embeddedMode/testDMLOpsInSnappy.bt index c116043427..102bdd3d4a 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/embeddedMode/testDMLOpsInSnappy.bt +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/embeddedMode/testDMLOpsInSnappy.bt @@ -18,3 +18,4 @@ io/snappydata/hydra/testDMLOps/embeddedMode/testDMLOpsInJob.conf C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 persistenceMode="sync" redundantCopies=1 + derbyDataLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind_derby" diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/jdbcClient/jdbcClient.inc b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/jdbcClient/jdbcClient.inc index ecd6838aa8..9572da0847 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/jdbcClient/jdbcClient.inc +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/jdbcClient/jdbcClient.inc @@ -48,7 +48,7 @@ INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf; INCLUDE $JTESTS/io/snappydata/hydra/testDMLOps/stopDerby.inc; hydra.Prms-totalTaskTimeSec = 900; -hydra.Prms-maxResultWaitSec = 600; +hydra.Prms-maxResultWaitSec = 900; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-testUniqueKeys = true; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-largeDataSet = true; diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nwTablesDDLWithPK.inc b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nwTablesDDLWithPK.inc index 5628b3a427..9abfc3d8cf 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nwTablesDDLWithPK.inc +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nwTablesDDLWithPK.inc @@ -1,13 +1,13 @@ io.snappydata.hydra.testDMLOps.SnappySchemaPrms-createTablesStatements = -"create table app.regions (RegionID int NOT NULL PRIMARY KEY,RegionDescription varchar(100),tid int)" -"create table app.categories (CategoryID int NOT NULL PRIMARY KEY,CategoryName varchar(100),Description varchar(100),Picture varchar(300),tid int)" -"create table app.shippers (ShipperID int NOT NULL PRIMARY KEY,CompanyName varchar(100) not null,Phone varchar(100),tid int)" -"create table app.employees(EmployeeID int NOT NULL PRIMARY KEY,LastName varchar(100),FirstName varchar(100),Title varchar(100),TitleOfCourtesy varchar(100),BirthDate timestamp,HireDate timestamp,Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100), HomePhone varchar(100),Extension varchar(100),Photo varchar(300),Notes varchar(300),ReportsTo int,PhotoPath varchar(100),tid int)" -"create table app.customers(CustomerID varchar(100) NOT NULL PRIMARY KEY,CompanyName varchar(100) ,ContactName varchar (100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),tid int)" -"create table app.orders (OrderID int NOT NULL PRIMARY KEY,CustomerID varchar(20),EmployeeID int,OrderDate timestamp,RequiredDate timestamp,ShippedDate timestamp,ShipVia int,Freight double,ShipName varchar(50),ShipAddress varchar(50),ShipCity varchar(15), ShipRegion varchar(20),ShipPostalCode varchar(10),ShipCountry varchar(20),tid int)" -"create table app.order_details (OrderID int NOT NULL PRIMARY KEY,ProductID int,UnitPrice double,Quantity smallint,Discount double,tid int)" -"create table app.products(ProductID int NOT NULL PRIMARY KEY,ProductName varchar(100),SupplierID int,CategoryID int,QuantityPerUnit varchar(100),UnitPrice double,UnitsInStock smallint,UnitsOnOrder smallint,ReorderLevel smallint,Discontinued smallint,tid int)" -"create table app.suppliers(SupplierID int NOT NULL PRIMARY KEY,CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),HomePage varchar(100),tid int) " -"create table app.territories(TerritoryID varchar(100) NOT NULL PRIMARY KEY,TerritoryDescription varchar(100),RegionID varchar(100),tid int)" -"create table app.employee_territories(EmployeeID int,TerritoryID varchar(100) NOT NULL PRIMARY KEY,tid int)" +"create table regions (RegionID int NOT NULL PRIMARY KEY,RegionDescription varchar(100),tid int)" +"create table categories (CategoryID int NOT NULL PRIMARY KEY,CategoryName varchar(100),Description varchar(100),Picture varchar(300),tid int)" +"create table shippers (ShipperID int NOT NULL PRIMARY KEY,CompanyName varchar(100) not null,Phone varchar(100),tid int)" +"create table employees(EmployeeID int NOT NULL PRIMARY KEY,LastName varchar(100),FirstName varchar(100),Title varchar(100),TitleOfCourtesy varchar(100),BirthDate timestamp,HireDate timestamp,Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100), HomePhone varchar(100),Extension varchar(100),Photo varchar(300),Notes varchar(300),ReportsTo int,PhotoPath varchar(100),tid int)" +"create table customers(CustomerID varchar(100) NOT NULL PRIMARY KEY,CompanyName varchar(100) ,ContactName varchar (100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),tid int)" +"create table orders (OrderID int NOT NULL PRIMARY KEY,CustomerID varchar(20),EmployeeID int,OrderDate timestamp,RequiredDate timestamp,ShippedDate timestamp,ShipVia int,Freight double,ShipName varchar(50),ShipAddress varchar(50),ShipCity varchar(15), ShipRegion varchar(20),ShipPostalCode varchar(10),ShipCountry varchar(20),tid int)" +"create table order_details (OrderID int NOT NULL PRIMARY KEY,ProductID int,UnitPrice double,Quantity smallint,Discount double,tid int)" +"create table products(ProductID int NOT NULL PRIMARY KEY,ProductName varchar(100),SupplierID int,CategoryID int,QuantityPerUnit varchar(100),UnitPrice double,UnitsInStock smallint,UnitsOnOrder smallint,ReorderLevel smallint,Discontinued smallint,tid int)" +"create table suppliers(SupplierID int NOT NULL PRIMARY KEY,CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),HomePage varchar(100),tid int) " +"create table territories(TerritoryID varchar(100) NOT NULL PRIMARY KEY,TerritoryDescription varchar(100),RegionID varchar(100),tid int)" +"create table employee_territories(EmployeeID int,TerritoryID varchar(100) NOT NULL PRIMARY KEY,tid int)" ; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nw_schema.inc b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nw_schema.inc index a497e7de2a..723e8a405b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nw_schema.inc +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/nw_schema.inc @@ -1,33 +1,33 @@ -io.snappydata.hydra.testDMLOps.SnappySchemaPrms-createSchemas = "create schema app"; +// io.snappydata.hydra.testDMLOps.SnappySchemaPrms-createSchemas = "create schema ; -io.snappydata.hydra.testDMLOps.SnappySchemaPrms-dropSchemas = "drop schema app cascade"; +// io.snappydata.hydra.testDMLOps.SnappySchemaPrms-dropSchemas = "drop schema app cascade"; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-createTablesStatements = -"create table app.regions (RegionID int,RegionDescription varchar(100),tid int)" -"create table app.categories (CategoryID int,CategoryName varchar(100),Description varchar(100),Picture varchar(300),tid int)" -"create table app.shippers (ShipperID int not null,CompanyName varchar(100) not null,Phone varchar(100),tid int)" -"create table app.employees(EmployeeID int,LastName varchar(100),FirstName varchar(100),Title varchar(100),TitleOfCourtesy varchar(100),BirthDate timestamp,HireDate timestamp,Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100), HomePhone varchar(100),Extension varchar(100),Photo varchar(300),Notes varchar(300),ReportsTo int,PhotoPath varchar(100),tid int)" -"create table app.customers(CustomerID varchar(100),CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),tid int)" -"create table app.orders (OrderID int,CustomerID varchar(20),EmployeeID int,OrderDate timestamp,RequiredDate timestamp,ShippedDate timestamp,ShipVia int,Freight double,ShipName varchar(50),ShipAddress varchar(50),ShipCity varchar(15), ShipRegion varchar(20),ShipPostalCode varchar(10),ShipCountry varchar(20),tid int)" -"create table app.order_details (OrderID int,ProductID int,UnitPrice double,Quantity smallint,Discount double,tid int)" -"create table app.products(ProductID int,ProductName varchar(100),SupplierID int,CategoryID int,QuantityPerUnit varchar(100),UnitPrice double,UnitsInStock smallint,UnitsOnOrder smallint,ReorderLevel smallint,Discontinued smallint,tid int)" -"create table app.suppliers(SupplierID int,CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),HomePage varchar(100),tid int) " -"create table app.territories(TerritoryID varchar(100),TerritoryDescription varchar(100),RegionID varchar(100),tid int)" -"create table app.employee_territories(EmployeeID int,TerritoryID varchar(100),tid int)" +"create table regions (RegionID int,RegionDescription varchar(100),tid int)" +"create table categories (CategoryID int,CategoryName varchar(100),Description varchar(100),Picture varchar(300),tid int)" +"create table shippers (ShipperID int not null,CompanyName varchar(100) not null,Phone varchar(100),tid int)" +"create table employees(EmployeeID int,LastName varchar(100),FirstName varchar(100),Title varchar(100),TitleOfCourtesy varchar(100),BirthDate timestamp,HireDate timestamp,Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100), HomePhone varchar(100),Extension varchar(100),Photo varchar(300),Notes varchar(300),ReportsTo int,PhotoPath varchar(100),tid int)" +"create table customers(CustomerID varchar(100),CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),tid int)" +"create table orders (OrderID int,CustomerID varchar(20),EmployeeID int,OrderDate timestamp,RequiredDate timestamp,ShippedDate timestamp,ShipVia int,Freight double,ShipName varchar(50),ShipAddress varchar(50),ShipCity varchar(15), ShipRegion varchar(20),ShipPostalCode varchar(10),ShipCountry varchar(20),tid int)" +"create table order_details (OrderID int,ProductID int,UnitPrice double,Quantity smallint,Discount double,tid int)" +"create table products(ProductID int,ProductName varchar(100),SupplierID int,CategoryID int,QuantityPerUnit varchar(100),UnitPrice double,UnitsInStock smallint,UnitsOnOrder smallint,ReorderLevel smallint,Discontinued smallint,tid int)" +"create table suppliers(SupplierID int,CompanyName varchar(100),ContactName varchar(100),ContactTitle varchar(100),Address varchar(100),City varchar(100),Region varchar(100),PostalCode varchar(100),Country varchar(100),Phone varchar(100),Fax varchar(100),HomePage varchar(100),tid int) " +"create table territories(TerritoryID varchar(100),TerritoryDescription varchar(100),RegionID varchar(100),tid int)" +"create table employee_territories(EmployeeID int,TerritoryID varchar(100),tid int)" ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-tablesList = -"app.regions" -"app.categories" -"app.shippers" -"app.suppliers" -"app.territories" -"app.employee_territories" -"app.products" -"app.employees" -"app.customers" -"app.orders" -"app.order_details" +"regions" +"categories" +"shippers" +"suppliers" +"territories" +"employee_territories" +"products" +"employees" +"customers" +"orders" +"order_details" ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-dataFileLocation = "/export/shared/QA_DATA/NW_1GB"; @@ -51,11 +51,11 @@ io.snappydata.hydra.testDMLOps.SnappySchemaPrms-loadDataInParts=false; //io.snappydata.hydra.testDMLOps.SnappySchemaPrms-numPartsForDataFiles=3; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-dmlTables = -"app.products" -"app.employees" -"app.customers" -"app.orders" -"app.order_details" +"products" +"employees" +"customers" +"orders" +"order_details" ; /* @@ -79,36 +79,36 @@ io.snappydata.hydra.testDMLOps.SnappySchemaPrms-insertCsvFileNames = ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-insertStmtsNonDMLTables = -"insert into app.regions(RegionID,RegionDescription,tid) values (?,?,?)" -"insert into app.categories(CategoryID,CategoryName,Description,Picture,tid) values (?,?,?,?,?)" -"insert into app.shippers(ShipperID,CompanyName,Phone,tid) values (?,?,?,?)" -"insert into app.suppliers(SupplierID,CompanyName,ContactName,ContactTitle,Address,City,Region,PostalCode,Country,Phone,Fax,HomePage,tid) values (?,?,?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.territories(TerritoryID,TerritoryDescription,RegionID,tid) values (?,?,?,?)" -"insert into app.employee_territories(EmployeeID,TerritoryID,tid) values (?,?,?)" +"insert into regions(RegionID,RegionDescription,tid) values (?,?,?)" +"insert into categories(CategoryID,CategoryName,Description,Picture,tid) values (?,?,?,?,?)" +"insert into shippers(ShipperID,CompanyName,Phone,tid) values (?,?,?,?)" +"insert into suppliers(SupplierID,CompanyName,ContactName,ContactTitle,Address,City,Region,PostalCode,Country,Phone,Fax,HomePage,tid) values (?,?,?,?,?,?,?,?,?,?,?,?,?)" +"insert into territories(TerritoryID,TerritoryDescription,RegionID,tid) values (?,?,?,?)" +"insert into employee_territories(EmployeeID,TerritoryID,tid) values (?,?,?)" ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-insertStmts = -"insert into app.products(ProductID,ProductName,SupplierID,CategoryID,QuantityPerUnit,UnitPrice,UnitsInStock,UnitsOnOrder,ReorderLevel,Discontinued,tid) values (?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.employees (EmployeeID,LastName,FirstName,Title,TitleOfCourtesy,BirthDate,HireDate,Address,City,Region,PostalCode,Country,HomePhone,Extension,Photo,Notes,ReportsTo,PhotoPath,tid) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.customers (CustomerID,CompanyName,ContactName,ContactTitle,Address,City,Region,PostalCode,Country,Phone,Fax,tid) values (?,?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.orders (OrderID,CustomerID,EmployeeID,OrderDate,RequiredDate,ShippedDate,ShipVia,Freight,ShipName,ShipAddress,ShipCity,ShipRegion,ShipPostalCode,ShipCountry,tid) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" -"insert into app.order_details (OrderID,ProductID,UnitPrice,Quantity,Discount,tid) values (?,?,?,?,?,?)" +"insert into products(ProductID,ProductName,SupplierID,CategoryID,QuantityPerUnit,UnitPrice,UnitsInStock,UnitsOnOrder,ReorderLevel,Discontinued,tid) values (?,?,?,?,?,?,?,?,?,?,?)" +"insert into employees (EmployeeID,LastName,FirstName,Title,TitleOfCourtesy,BirthDate,HireDate,Address,City,Region,PostalCode,Country,HomePhone,Extension,Photo,Notes,ReportsTo,PhotoPath,tid) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +"insert into customers (CustomerID,CompanyName,ContactName,ContactTitle,Address,City,Region,PostalCode,Country,Phone,Fax,tid) values (?,?,?,?,?,?,?,?,?,?,?,?)" +"insert into orders (OrderID,CustomerID,EmployeeID,OrderDate,RequiredDate,ShippedDate,ShipVia,Freight,ShipName,ShipAddress,ShipCity,ShipRegion,ShipPostalCode,ShipCountry,tid) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +"insert into order_details (OrderID,ProductID,UnitPrice,Quantity,Discount,tid) values (?,?,?,?,?,?)" ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-updateStmts = -"UPDATE app.products SET UnitPrice = UnitPrice * 1.1 WHERE CategoryID = (SELECT max(CategoryID) from products where tid=$tid)" -"UPDATE app.products SET UnitPrice = UnitPrice * 0.9" -"UPDATE app.products SET UnitPrice = UnitPrice * 1.1, ReorderLevel = cast((ReorderLevel + 5) as smallint) WHERE CategoryID = (SELECT min(CategoryID) from products where tid=$tid)" -"UPDATE app.products SET Discontinued=cast(-1 as smallint) where Discontinued=1" -"UPDATE app.products SET Discontinued=cast(-1 as smallint) WHERE Discontinued=-1" -"UPDATE app.order_details SET UnitPrice = UnitPrice * 1.1" -"UPDATE app.order_details SET UnitPrice = UnitPrice * 1.1 WHERE ORDERID < 10450" -"UPDATE app.order_details SET Quantity = cast(0 as smallint) WHERE ProductID = 11" -"UPDATE app.employees SET TitleOfCourtesy='Mr.' WHERE FIRSTNAME LIKE '%3%'" -"UPDATE app.employees SET TitleOfCourtesy='Mrs.' WHERE FIRSTNAME LIKE '%3%'" -"UPDATE app.orders SET ShipVia = ShipVia + 1 WHERE OrderID IN (SELECT max(orderID) from orders where tid=$tid)" -"UPDATE app.orders SET Freight = Freight * 1.1 WHERE Freight >= 100" -"UPDATE app.customers SET CompanyName='abc_$tid'" +"UPDATE products SET UnitPrice = UnitPrice * 1.1 WHERE CategoryID = (SELECT max(CategoryID) from products where tid=$tid)" +"UPDATE products SET UnitPrice = UnitPrice * 0.9" +"UPDATE products SET UnitPrice = UnitPrice * 1.1, ReorderLevel = cast((ReorderLevel + 5) as smallint) WHERE CategoryID = (SELECT min(CategoryID) from products where tid=$tid)" +"UPDATE products SET Discontinued=cast(-1 as smallint) where Discontinued=1" +"UPDATE products SET Discontinued=cast(-1 as smallint) WHERE Discontinued=-1" +"UPDATE order_details SET UnitPrice = UnitPrice * 1.1" +"UPDATE order_details SET UnitPrice = UnitPrice * 1.1 WHERE ORDERID < 10450" +"UPDATE order_details SET Quantity = cast(0 as smallint) WHERE ProductID = 11" +"UPDATE employees SET TitleOfCourtesy='Mr.' WHERE FIRSTNAME LIKE '%3%'" +"UPDATE employees SET TitleOfCourtesy='Mrs.' WHERE FIRSTNAME LIKE '%3%'" +"UPDATE orders SET ShipVia = ShipVia + 1 WHERE OrderID IN (SELECT max(orderID) from orders where tid=$tid)" +"UPDATE orders SET Freight = Freight * 1.1 WHERE Freight >= 100" +"UPDATE customers SET CompanyName='abc_$tid'" ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-afterUpdateSelects = @@ -128,35 +128,35 @@ io.snappydata.hydra.testDMLOps.SnappySchemaPrms-afterUpdateSelects = ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-updateTables = -"app.products" -"app.products" -"app.products" -"app.products" -"app.products" -"app.order_details" -"app.order_details" -"app.order_details" -"app.employees" -"app.employees" -"app.orders" -"app.orders" -"app.customers" +"products" +"products" +"products" +"products" +"products" +"order_details" +"order_details" +"order_details" +"employees" +"employees" +"orders" +"orders" +"customers" ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-deleteStmts = -"delete from app.orders where orderid IN (SELECT max(orderID) from orders where tid=$tid)" -"delete from app.order_details where Quantity < 1 " -"delete from app.orders where orderid IN (SELECT min(orderID) from orders where tid=$tid)" -"delete from app.customers where CustomerID IN (SELECT min(CustomerID) from customers where tid=$tid)" -"delete from app.customers where CustomerID IN (SELECT max(CustomerID) from customers where tid=$tid)" -"delete from app.employees where EmployeeID = 6" -"delete from app.employees where EmployeeID < 5" -"delete from app.employees where FIRSTNAME LIKE '%2'" +"delete from orders where orderid IN (SELECT max(orderID) from orders where tid=$tid)" +"delete from order_details where Quantity < 1 " +"delete from orders where orderid IN (SELECT min(orderID) from orders where tid=$tid)" +"delete from customers where CustomerID IN (SELECT min(CustomerID) from customers where tid=$tid)" +"delete from customers where CustomerID IN (SELECT max(CustomerID) from customers where tid=$tid)" +"delete from employees where EmployeeID = 6" +"delete from employees where EmployeeID < 5" +"delete from employees where FIRSTNAME LIKE '%2'" ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-afterDeleteSelects = -"select * from app.orders" -"select * from app.order_details" +"select * from orders" +"select * from order_details" "select * from orders" "select * from customers" "select * from customers" @@ -166,12 +166,12 @@ io.snappydata.hydra.testDMLOps.SnappySchemaPrms-afterDeleteSelects = ; io.snappydata.hydra.testDMLOps.SnappySchemaPrms-deleteTables = -"app.orders" -"app.order_details" -"app.orders" -"app.customers" -"app.customers" -"app.employees" -"app.employees" -"app.employees" +"orders" +"order_details" +"orders" +"customers" +"customers" +"employees" +"employees" +"employees" ; diff --git a/dtests/src/test/scala/io/snappydata/hydra/alterTable/AlterTablesJob.scala b/dtests/src/test/scala/io/snappydata/hydra/alterTable/AlterTablesJob.scala index 4285f909ba..4e91e422bb 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/alterTable/AlterTablesJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/alterTable/AlterTablesJob.scala @@ -30,21 +30,23 @@ class AlterTablesJob extends SnappySQLJob { override def runSnappyJob(snSession: SnappySession, jobConfig: Config): Any = { val pw = new PrintWriter(new FileOutputStream(new File("AlterTablesJob.out"), true)); Try { + val snc = snSession.sqlContext // scalastyle:off println pw.println(s"Alter tables Test started at : " + System.currentTimeMillis) + // snc.setSchema("gemfire1") snc.alterTable("orders", true, StructField("FirstName", StringType, true)) var query = "SELECT * FROM orders" var result = snc.sql(query).schema.fields.length pw.println("schema fields length for orders" + "\nResults : " + result) assert(snc.sql("SELECT * FROM " + "orders").schema.fields.length == 15) - snc.sql("insert into app.orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000'," + + snc.sql("insert into orders values(10251,'AROUT',4,'1996-07-08 00:00:00.000'," + "'1996-08-05 00:00:00.000','1996-07-15 00:00:00.000',4,41.34,'Victuailles en stock 2', " + "'rue du Commerce','Lyon',NULL,'69004','France','empFirstName2')"); - snc.sql("insert into app.orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000'," + + snc.sql("insert into orders values(10260,'CENTC',13,'1996-07-19 00:00:00.000'," + "'1996-08-16 00:00:00.000','1996-07-29 00:00:00.000',13,55.09,'Ottilies Kaoseladen'," + "'Mehrheimerstr. 369','Kaqln',NULL,'50739','Germany','empFirstName4')"); - snc.sql("insert into app.orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000'," + + snc.sql("insert into orders values(10265,'DUMON',18,'1996-07-25 00:00:00.000'," + "'1996-08-22 00:00:00.000','1996-08-12 00:00:00.000',18,55.28," + "'Blondel pare et fils 24', 'place Klacber','Strasbourg',NULL," + "'67000','France','empFirstName5')"); @@ -68,9 +70,9 @@ class AlterTablesJob extends SnappySQLJob { result = snc.sql(query).schema.fields.length pw.println("Query : " + query + "\nResults : " + result) assert(result == 6) - snc.sql("insert into app.order_details values(10250,72,34.8,5,0,'custID4')"); - snc.sql("insert into app.order_details values(10249,42,9.8,10,0,'custID14')"); - snc.sql("insert into app.order_details values(10253,41,7.7,10,0,'custID7')"); + snc.sql("insert into order_details values(10250,72,34.8,5,0,'custID4')"); + snc.sql("insert into order_details values(10249,42,9.8,10,0,'custID14')"); + snc.sql("insert into order_details values(10253,41,7.7,10,0,'custID7')"); query1 = "SELECT * FROM order_details where CustomerID='custID4'"; result1 = snc.sql(query1); pw.println("Query : " + query1 + "\nResult : " + result1.collect() + "\n " + result1.show()) diff --git a/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala b/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala index 7854f39f95..65edf7db64 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala @@ -138,6 +138,22 @@ object CDCValidationApp { } if (flag.equals(2.toString)) { + // Match the difference : + pw.println("=================================================Count Validation===================================================") + pw.println("For table " + tableName + " Present sqlServer cnt = " + sqlServerTableCnt + " Previous sqlServer cnt " + sqlServerTableCnt1Arr(j)) + pw.println("For table " + tableName + " Present snappy cnt = " + snappyTableCnt + " Previous snappy cnt " + snappyTableCnt1Arr(j)) + val sqlServerCntDiff = sqlServerTableCnt - sqlServerTableCnt1Arr(j).toLong + val snappyCntDiff = snappyTableCnt - snappyTableCnt1Arr(j).toLong + + if (sqlServerCntDiff == snappyCntDiff) { + pw.println(s"SUCCESS :$tableName in snappy cluster has equal difference of $snappyCntDiff before and after ingestion ") + pw.println(s"SUCCESS :$tableName in sqlServer cluster has equal difference of $sqlServerCntDiff before and after ingestion") + } + else { + pw.println("FAILURE:" + tableName + " table has incorrect number of records: sqlServer table has " + + sqlServerTableCnt + " and snappytable has " + snappyTableCnt) + } + pw.println("====================================================================================================================") //Do full resultSet validation: pw.println() @@ -156,22 +172,7 @@ object CDCValidationApp { checkDataConsistency(endRange) //fullResultSetValidation(snc, connection, sqlQ, snappyQ, pw) pw.println("=====================================================================================================================") - // Match the difference : - pw.println("=================================================Count Validation=====================================================") - pw.println("For table " + tableName + " Present sqlServer cnt = " + sqlServerTableCnt + " Previous sqlServer cnt " + sqlServerTableCnt1Arr(j)) - pw.println("For table " + tableName + " Present snappy cnt = " + snappyTableCnt + " Previous snappy cnt " + snappyTableCnt1Arr(j)) - val sqlServerCntDiff = sqlServerTableCnt - sqlServerTableCnt1Arr(j).toLong - val snappyCntDiff = snappyTableCnt - snappyTableCnt1Arr(j).toLong - if (sqlServerCntDiff == snappyCntDiff) { - pw.println(s"SUCCESS :$tableName in snappy cluster has equal difference of $snappyCntDiff before and after ingestion ") - pw.println(s"SUCCESS :$tableName in sqlServer cluster has equal difference of $sqlServerCntDiff before and after ingestion") - } - else { - pw.println("FAILURE:" + tableName + " table has incorrect number of records: sqlServer table has " - + sqlServerTableCnt + " and snappytable has " + snappyTableCnt) - } - pw.println("=====================================================================================================================") pw.println() } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSnappyJob.scala b/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSnappyJob.scala index 3f33dace12..2f5cf8ddd4 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSnappyJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSnappyJob.scala @@ -35,7 +35,6 @@ object CreateAndLoadTablesSnappyJob extends SnappySQLJob { val queryArray = scala.io.Source.fromFile(queryFile).getLines().mkString.split(";") val schema1: String = jobConfig.getString("schema1") val schema2: String = jobConfig.getString("schema2") - // val passfile: String = jobConfig.getString("passfile1") val isGrant: Boolean = jobConfig.getBoolean("isGrant") val expectedCntWithGrant: Int = jobConfig.getInt("expectedCntWithGrant") val unExpectedCntWithGrant: Int = jobConfig.getInt("unExpectedCntWithGrant") @@ -48,50 +47,22 @@ object CreateAndLoadTablesSnappyJob extends SnappySQLJob { val props = Map[String, String]() def getCurrentDirectory = new java.io.File(".").getCanonicalPath - // scalastyle:off println val pw = new PrintWriter("SnappyJobOutPut.out") Try { snc.setConf("dataFilesLocation", dataFilesLocation) NWQueries.snc = snc NWQueries.dataFilesLocation = dataFilesLocation - SecurityTestUtil.createColRowTables(snc) + // SecurityTestUtil.createColRowTables(snc) SecurityTestUtil.runQueries(snc, queryArray, expectedCntWithGrant, unExpectedCntWithGrant, isGrant, userSchema, isSelect, pw) - runQuery() - /* snc.sql("select * from user2.airline limit 10").show() - println("SELECT successful") - snc.sql("UPDATE user2.airline SET DEST = 'INDIA' WHERE UNIQUECARRIER = 'WN' ") - println("UPDATE successful") - snc.sql("select uniquecarrier,dest from user2.airline where dest = 'INDIA '").show() -*/ - /* snc.sql("DELETE FROM user2.airline WHERE DEST = 'INDIA' AND UNIQUECARRIER = 'WN' ").show() - snc.sql("select uniquecarrier,dest from user2.airline where dest = 'INDIA '").show() */ } match { case Success(v) => pw.close() s"See ${getCurrentDirectory}/SnappyJobOutPut.out" case Failure(e) => pw.close(); throw e; - // println("Got Exception " + ex.getMessage()) - /* case ex : Exception => { - /* if (isAuth) { - unExpectedExpCnt = unExpectedExpCnt + 1 - println("Got unExpected Exception " + ex.printStackTrace()) - println("unExpectedExpCnt = " + unExpectedExpCnt) - } - else { - expectedExpCnt = expectedExpCnt + 1 - println("Got Expected exception " + ex.printStackTrace()) - println("expectedCnt = " + expectedExpCnt) - } */ - println("Got Exception " + ex.getMessage()) - } */ } - def runQuery() : Unit = { - println("Inside runQuery") - - } } override def isValidJob(sc: SnappySession, config: Config): SnappyJobValidation = SnappyJobValid() diff --git a/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala b/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala index fe68fc4175..262291a557 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala @@ -38,13 +38,6 @@ object CreateAndLoadTablesSparkApp { val pw = new PrintWriter(new FileOutputStream(new File("CreateAndLoadNWTablesSparkApp.out"), true)); - val dataFilesLocation = args(0) - snc.setConf("dataFilesLocation", dataFilesLocation) - NWQueries.snc = snc - NWQueries.dataFilesLocation = dataFilesLocation - pw.println(s"dataFilesLocation : ${dataFilesLocation}") - NWTestUtil.dropTables(snc) - // SecurityTestUtil.createColRowTables(snc) println("Getting users arguments") val queryFile = args(1) val queryArray = scala.io.Source.fromFile(queryFile).getLines().mkString.split(";") diff --git a/dtests/src/test/scala/io/snappydata/hydra/security/SecurityTestUtil.scala b/dtests/src/test/scala/io/snappydata/hydra/security/SecurityTestUtil.scala index b72744ea3e..325fbb0e80 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/security/SecurityTestUtil.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/security/SecurityTestUtil.scala @@ -19,8 +19,12 @@ package io.snappydata.hydra.security import java.io.{PrintWriter} +import scala.collection.immutable.HashMap + import io.snappydata.hydra.northwind.NWQueries +import org.apache.spark.sql.{DataFrame, SnappyContext} + import org.apache.spark.sql.SnappyContext object SecurityTestUtil { @@ -28,6 +32,10 @@ object SecurityTestUtil { var expectedExpCnt = 0; var unExpectedExpCnt = 0; + var policyUserMap: HashMap[String, String] = HashMap() + var policySelectQueryMap: HashMap[Map[String, DataFrame], String] = HashMap() + var policyFullSelectQueryMap: HashMap[Map[String, DataFrame], String] = HashMap() + def createColRowTables(snc: SnappyContext) : Unit = { println("Inside createColAndRow tables") snc.sql(NWQueries.categories_table) @@ -63,6 +71,157 @@ object SecurityTestUtil { println("Finished createColAndRow tables") } + def createPolicy(snc: SnappyContext, userSchema: Array[String], + userName: Array[String]): Unit = { + println("Inside createPolicy() ") + var filterCond, orderBy = "" + var cnt = 1 + try { + for (i <- 0 to userName.length - 1) { + val policyUser = userName(i) + for (s <- 0 to userSchema.length - 1) { + if (userSchema(s).contains("employees")) { + filterCond = "EMPLOYEEID = 1 AND COUNTRY = 'USA'" + orderBy = " ORDER BY EMPLOYEEID asc " + } + else { + filterCond = "CATEGORYID = 4" + orderBy = " ORDER BY CATEGORYID asc " + } + + val grantSql = " GRANT SELECT on " + userSchema(s) + " TO " + userName(i) + println("GRANT sql statement is " + grantSql) + snc.sql(grantSql) + + var policyName = "p" + 0; + while (policyUserMap.contains(policyName)) { + println("checking for policy name " + policyName); + policyName = "p" + cnt + println("The next policy name is " + policyName) + cnt = cnt + 1 + } + var queryMap1: Map[String, DataFrame] = Map() + var queryMap2: Map[String, DataFrame] = Map() + val policyStr = "CREATE POLICY " + policyName + " ON " + userSchema(s) + + " FOR SELECT TO " + policyUser + " USING " + filterCond + println("The create policy string is " + policyStr) + val selectQryWF = "SELECT * FROM " + userSchema(s) + " WHERE " + filterCond; + println("Select Query with filter is " + selectQryWF) + val selectQry = "SELECT * FROM " + userSchema(s) + orderBy; + println("Select Query is " + selectQry) + println("Policy created for " + policyUser + " on table " + + userSchema(s) + " is " + policyStr) + snc.sql(policyStr) + val result1 = snc.sql(selectQry) + val result2 = snc.sql(selectQryWF) + + queryMap1 += (selectQry -> result1) + queryMap2 += (selectQryWF -> result2) + policySelectQueryMap += (queryMap2 -> policyUser) + policyFullSelectQueryMap += (queryMap1 -> policyUser) + policyUserMap += (policyName -> userSchema(s)) + } + } + } + catch { + case ex: Exception => { + println("Caught exception in createPolicy " + ex.printStackTrace()) + } + } + println("Successfully completed createPolicy task") + } + + def dropPolicy(snc: SnappyContext): Unit = { + println("Inside dropPolicy()") + try { + val policyMap = policyUserMap + for ((k, v) <- policyMap) { + val policyNm = k + val schemaOwner = v + val dropPolicyStr = " DROP POLICY " + policyNm + println("Dropping policy p " + policyNm + " with schemaOwner " + schemaOwner) + snc.sql(dropPolicyStr) + } + } + catch { + case ex: Exception => { + println("Caught exception in dropPolicy() " + ex.printStackTrace()) + } + } + } + + def enableRLS(snc: SnappyContext, userSchema: Array[String]): Unit = { + println("Inside enableRLS()") + val isAltrTableRLS = true + var alterTabSql = "" + try { + for (s <- 0 to userSchema.length - 1) { + if (isAltrTableRLS) { + alterTabSql = "ALTER TABLE " + userSchema(s) + " ENABLE ROW LEVEL SECURITY"; + println("The alter table sql is " + alterTabSql); + } else { + alterTabSql = "ALTER TABLE " + userSchema(s) + " DISABLE ROW LEVEL SECURITY"; + println("The alter table sql is " + alterTabSql); + } + snc.sql(alterTabSql) + } + } + catch { + case ex: Exception => { + println("Caught exception in enableRLS " + ex.printStackTrace()) + } + } + println("Successfully completed enableRLS task") + } + + def validateRLS(snc: SnappyContext, isDropPolicy: Boolean, isAltrTableRLS: + Boolean = true): Unit = { + println("Inside validateRLS") + var queryUserMap: HashMap[Map[String, DataFrame], String] = HashMap() + if (isDropPolicy || !isAltrTableRLS) { + queryUserMap = policyFullSelectQueryMap; // this map contains rs fro select * from table query + println("Using policyFullSelectQueryMap"); + } else { + queryUserMap = policySelectQueryMap; // this map contains rs fro + // select * from table with condition query + println("Using policySelectQueryMap"); + } + try + for ((k, v) <- queryUserMap) { + val queryMap = k + val policyUser = v + println("The user is " + policyUser); + for ((k1, v1) <- queryMap) { + val selectQry = k1 + val finalSelectQ: Array[String] = selectQry.split("WHERE") + val prevDF = v1 + println("The select Query is " + selectQry); + println("The Final select Query tobe executed by policy User is " + finalSelectQ(0)); + val currDF = snc.sql(finalSelectQ(0)) + val currFiltrDF = snc.sql(selectQry) + val currDFSize = currDF.count() + val prevDFSize = prevDF.count() + val currFiltrDFSize = currFiltrDF.count() + println("Current DF size = " + currDFSize + " Previous DF size = " + prevDFSize + + " Current DF with filter size = " + currFiltrDFSize) + if ((currDFSize == prevDFSize) && (currFiltrDFSize == prevDFSize)) { + println(" The number of rows returned are equal ") + } + else { + println(" The number of rows returned donot match Current DF size = " + currDFSize + + " Previous DF size = " + prevDFSize + " and CurrFiltrDF Size = " + + currFiltrDFSize) + } + } + } + + catch { + case ex: Exception => { + println("Caught exception in validateRLS " + ex.printStackTrace()) + } + } + } + def runQueries(snc: SnappyContext, queryArray: Array[String], expectExcpCnt: Integer, unExpectExcpCnt: Integer, isGrant: Boolean, userSchema: Array[String], isSelect: Boolean, pw: PrintWriter) diff --git a/dtests/src/test/scala/io/snappydata/hydra/security/rls/RLSSnappyJob.scala b/dtests/src/test/scala/io/snappydata/hydra/security/rls/RLSSnappyJob.scala new file mode 100644 index 0000000000..57f865c4b7 --- /dev/null +++ b/dtests/src/test/scala/io/snappydata/hydra/security/rls/RLSSnappyJob.scala @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2017 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package io.snappydata.hydra.security.rls + +import java.io.PrintWriter + +import scala.util.{Failure, Success, Try} + +import com.pivotal.gemfirexd.Attribute +import com.typesafe.config.Config +import io.snappydata.hydra.security.SecurityTestUtil + +import org.apache.spark.sql.{SnappyJobValid, SnappyJobValidation, SnappySQLJob, SnappySession} + +object RLSSnappyJob extends SnappySQLJob { + // scalastyle:off println + println(" Inside RLSSnappyJob") + override def runSnappyJob(snSession: SnappySession, jobConfig: Config): Any = { + val snc = snSession.sqlContext + // val dataFilesLocation = jobConfig.getString("dataFilesLocation") + val schema1: String = jobConfig.getString("schema1") + val schema2: String = jobConfig.getString("schema2") + val user2: String = jobConfig.getString("gemfire2") + val user3: String = jobConfig.getString("user3") + val user4: String = jobConfig.getString("user4") + val userSchema = new Array[String](2) + userSchema(0) = schema1; + userSchema(1) = schema2; + val userNames = new Array[String](2) + userNames(0) = user3; + userNames(1) = user4; + + val snc2 = snc.newSession() + snc2.snappySession.conf.set(Attribute.USERNAME_ATTR, user2) + snc2.snappySession.conf.set(Attribute.PASSWORD_ATTR, user2) + + val snc3 = snc.newSession() + snc3.snappySession.conf.set(Attribute.USERNAME_ATTR, user3) + snc3.snappySession.conf.set(Attribute.PASSWORD_ATTR, user3) + + val snc4 = snc.newSession() + snc4.snappySession.conf.set(Attribute.USERNAME_ATTR, user4) + snc4.snappySession.conf.set(Attribute.PASSWORD_ATTR, user4) + + def getCurrentDirectory = new java.io.File(".").getCanonicalPath + + // scalastyle:off println + val pw = new PrintWriter("RLSSnappyJob.out") + Try { + SecurityTestUtil.createPolicy(snc2, userSchema, userNames) + SecurityTestUtil.enableRLS(snc2, userSchema) + SecurityTestUtil.validateRLS(snc3, false) + SecurityTestUtil.validateRLS(snc4, false) + SecurityTestUtil.dropPolicy(snc2) + SecurityTestUtil.validateRLS(snc3, true) + SecurityTestUtil.validateRLS(snc4, true) + } + match { + case Success(v) => pw.close() + s"See ${getCurrentDirectory}/RLSSnappyJob.out" + case Failure(e) => pw.close(); + throw e; + } + def runQuery() : Unit = { + println("Inside runQuery") + } + } + override def isValidJob(sc: SnappySession, config: Config): SnappyJobValidation = SnappyJobValid() +} + diff --git a/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ConvertCsvToParquetAndLoadTablesJob.scala b/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ConvertCsvToParquetAndLoadTablesJob.scala index 37174264f1..1b265da3db 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ConvertCsvToParquetAndLoadTablesJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/testDMLOps/ConvertCsvToParquetAndLoadTablesJob.scala @@ -49,18 +49,30 @@ class ConvertCsvToParquetAndLoadTablesJob extends SnappySQLJob { SnappyDMLTestUtil.createParquetData(snc, parquetFileLocation, pw) snc.setConf("parquetFileLocation", parquetFileLocation) - + pw.println("Now loading data to the snappy tables...") + pw.println("Loading data into orders..."); pw.flush() SnappyDMLTestUtil.orders_par(snc).write.insertInto("orders") + pw.println("Loading data into order_details..."); pw.flush() SnappyDMLTestUtil.order_details_par(snc).write.insertInto("order_details") + pw.println("Loading data into orders..."); pw.flush() SnappyDMLTestUtil.regions_par(snc).write.insertInto("regions") + pw.println("Loading data into categories..."); pw.flush() SnappyDMLTestUtil.categories_par(snc).write.insertInto("categories") + pw.println("Loading data into shippers..."); pw.flush() SnappyDMLTestUtil.shippers_par(snc).write.insertInto("shippers") + pw.println("Loading data into employees..."); pw.flush() SnappyDMLTestUtil.employees_par(snc).write.insertInto("employees") + pw.println("Loading data into customers..."); pw.flush() SnappyDMLTestUtil.customers_par(snc).write.insertInto("customers") + pw.println("Loading data into products..."); pw.flush() SnappyDMLTestUtil.products_par(snc).write.insertInto("products") + pw.println("Loading data into suppliers..."); pw.flush() SnappyDMLTestUtil.suppliers_par(snc).write.insertInto("suppliers") + pw.println("Loading data into territories..."); pw.flush() SnappyDMLTestUtil.territories_par(snc).write.insertInto("territories") + pw.println("Loading data into employee_territories..."); pw.flush() SnappyDMLTestUtil.employee_territories_par(snc).write.insertInto("employee_territories") + pw.println("Done loading data into snappy tables.") } match { case Success(v) => pw.close()