diff --git a/MoquiConf.xml b/MoquiConf.xml index b9dac2a..246c168 100644 --- a/MoquiConf.xml +++ b/MoquiConf.xml @@ -2,13 +2,19 @@ - - + + + + + + + - + diff --git a/src/main/groovy/org/moqui/aws/S3ClientToolFactory.groovy b/src/main/groovy/org/moqui/aws/S3ClientToolFactory.groovy new file mode 100644 index 0000000..b29c48c --- /dev/null +++ b/src/main/groovy/org/moqui/aws/S3ClientToolFactory.groovy @@ -0,0 +1,73 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to the + * public domain worldwide. This software is distributed without any + * warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication + * along with this software (see the LICENSE.md file). If not, see + * . + */ +package org.moqui.aws + +import com.amazonaws.services.s3.AmazonS3 +import com.amazonaws.services.s3.AmazonS3ClientBuilder +import groovy.transform.CompileStatic +import org.moqui.context.ExecutionContextFactory +import org.moqui.context.ToolFactory +import org.moqui.util.SystemBinding +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** A ToolFactory for AWS S3 Client */ +@CompileStatic +class S3ClientToolFactory implements ToolFactory { + protected final static Logger logger = LoggerFactory.getLogger(S3ClientToolFactory.class) + final static String TOOL_NAME = "AwsS3Client" + + protected ExecutionContextFactory ecf = null + protected AmazonS3 s3Client = null + + /** Default empty constructor */ + S3ClientToolFactory() { } + + @Override String getName() { return TOOL_NAME } + + @Override + void init(ExecutionContextFactory ecf) { + this.ecf = ecf + // NOTE: minimal explicit configuration here, see: + // https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html + // https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-region-selection.html + + // There is no Java sys prop key for region, and env var vs Java sys prop keys are different for access key ID and secret + // so normalize here to the standard SDK env var keys and support from Java sys props as well + String awsRegion = SystemBinding.getPropOrEnv("AWS_REGION") + String awsAccessKeyId = SystemBinding.getPropOrEnv("AWS_ACCESS_KEY_ID") + String awsSecret = SystemBinding.getPropOrEnv("AWS_SECRET_ACCESS_KEY") + if (awsAccessKeyId && awsSecret) { + System.setProperty("aws.accessKeyId", awsAccessKeyId) + System.setProperty("aws.secretKey", awsSecret) + } + + logger.info("Starting AWS S3 Client with region ${awsRegion} access ID ${awsAccessKeyId}") + + AmazonS3ClientBuilder cb = AmazonS3ClientBuilder.standard() + if (awsRegion) cb.withRegion(awsRegion) + s3Client = cb.build() + } + + @Override AmazonS3 getInstance(Object... parameters) { return s3Client } + + @Override + void destroy() { + // stop Camel to prevent more calls coming in + if (s3Client != null) try { + s3Client.shutdown() + logger.info("AWS S3 Client shut down") + } catch (Throwable t) { logger.error("Error in AWS S3 Client shut down", t) } + } +} diff --git a/src/main/groovy/org/moqui/aws/S3ResourceReference.groovy b/src/main/groovy/org/moqui/aws/S3ResourceReference.groovy index ccf0617..62cc40b 100644 --- a/src/main/groovy/org/moqui/aws/S3ResourceReference.groovy +++ b/src/main/groovy/org/moqui/aws/S3ResourceReference.groovy @@ -13,22 +13,60 @@ */ package org.moqui.aws +import com.amazonaws.services.s3.AmazonS3 +import com.amazonaws.services.s3.model.AmazonS3Exception +import com.amazonaws.services.s3.model.CopyObjectRequest +import com.amazonaws.services.s3.model.GetObjectMetadataRequest +import com.amazonaws.services.s3.model.GetObjectRequest +import com.amazonaws.services.s3.model.ListObjectsV2Request +import com.amazonaws.services.s3.model.ListObjectsV2Result +import com.amazonaws.services.s3.model.ListVersionsRequest +import com.amazonaws.services.s3.model.ObjectMetadata +import com.amazonaws.services.s3.model.S3Object +import com.amazonaws.services.s3.model.S3ObjectInputStream +import com.amazonaws.services.s3.model.S3ObjectSummary +import com.amazonaws.services.s3.model.S3VersionSummary +import com.amazonaws.services.s3.model.VersionListing + +import groovy.transform.CompileStatic + import org.moqui.BaseArtifactException +import org.moqui.BaseException import org.moqui.impl.context.ExecutionContextFactoryImpl import org.moqui.impl.context.reference.BaseResourceReference import org.moqui.resource.ResourceReference +// NOTE: IDE says this isn't needed but compiler requires it +import org.moqui.resource.ResourceReference.Version import org.moqui.util.ObjectUtilities + import org.slf4j.Logger import org.slf4j.LoggerFactory -import javax.sql.rowset.serial.SerialBlob -import java.nio.charset.StandardCharsets +import java.sql.Timestamp + +// TODO: catch and wrap AmazonServiceException throughout? is a RuntimeException anyway +// TODO: need to worry about ResetException? would have to use temp files for all puts, see https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/best-practices.html + +/* +Handy Docs: +https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingTheMPJavaAPI.html +https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectOperations.html +https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/examples-s3-objects.html + +Important Classes: +https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html +https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/S3Object.html +https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/ObjectMetadata.html + */ +@CompileStatic class S3ResourceReference extends BaseResourceReference { protected final static Logger logger = LoggerFactory.getLogger(S3ResourceReference.class) - public final static String locationPrefix = "s3://" + public final static String locationPrefix = "aws3://" + public final static boolean autoCreateBucket = true String location + Boolean knownDirectory = (Boolean) null S3ResourceReference() { } @@ -37,6 +75,12 @@ class S3ResourceReference extends BaseResourceReference { this.location = location return this } + S3ResourceReference init(String location, Boolean knownDirectory, ExecutionContextFactoryImpl ecf) { + this.ecf = ecf + this.location = location + this.knownDirectory = knownDirectory + return this + } @Override ResourceReference createNew(String location) { S3ResourceReference resRef = new S3ResourceReference() @@ -45,20 +89,18 @@ class S3ResourceReference extends BaseResourceReference { } @Override String getLocation() { location } - String getBucketName() { - if (!location) return "" - // should have a prefix of "s3://" and then first path segment is bucket name + static String getBucketName(String location) { + if (!location) throw new BaseArtifactException("No location specified, cannot get bucket name (first path segment)") + // after prefix first path segment is bucket name String fullPath = location.substring(locationPrefix.length()) int slashIdx = fullPath.indexOf("/") - if (slashIdx) { - return fullPath.substring(0, slashIdx) - } else { - return fullPath - } + String bName = slashIdx == -1 ? fullPath : fullPath.substring(0, slashIdx) + if (!bName) throw new BaseArtifactException("No bucket name (first path segment) in location ${location}") + return bName } - String getPath() { + static String getPath(String location) { if (!location) return "" - // should have a prefix of "s3://" and then first path segment is bucket name + // after prefix first path segment is bucket name so strip that to get path String fullPath = location.substring(locationPrefix.length()) int slashIdx = fullPath.indexOf("/") if (slashIdx) { @@ -69,8 +111,20 @@ class S3ResourceReference extends BaseResourceReference { } @Override InputStream openStream() { - // TODO - return null + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + try { + S3Object obj = s3Client.getObject(bucketName, path) + S3ObjectInputStream s3is = obj.getObjectContent() + return s3is + } catch (AmazonS3Exception e) { + if (e.getStatusCode() == 404) { + logger.warn("Not found (404) error in openStream for bucket ${bucketName} path ${path}: ${e.toString()}") + return null + } else { throw e } + } } @Override OutputStream getOutputStream() { @@ -78,7 +132,20 @@ class S3ResourceReference extends BaseResourceReference { throw new UnsupportedOperationException("The getOutputStream method is not supported for s3 resources, use putStream() instead") } - @Override String getText() { return ObjectUtilities.getStreamText(openStream()) } + @Override String getText() { + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + try { + return s3Client.getObjectAsString(bucketName, path) + } catch (AmazonS3Exception e) { + if (e.getStatusCode() == 404) { + logger.warn("Not found (404) error in getText for bucket ${bucketName} path ${path}: ${e.toString()}") + return null + } else { throw e } + } + } @Override boolean supportsAll() { true } @@ -87,46 +154,128 @@ class S3ResourceReference extends BaseResourceReference { @Override boolean supportsDirectory() { true } @Override boolean isFile() { - // TODO - return true + if (knownDirectory != null) return !knownDirectory.booleanValue() + // NOTE how to determine? if exists is file should do for now + if (s3Client.doesObjectExist(getBucketName(location), getPath(location))) { + knownDirectory = Boolean.FALSE + return true + } else { + return false + } } @Override boolean isDirectory() { - if (!getPath()) return true // consider root a directory - // TODO - return false + // logger.warn("isDirectory loc ${location} knownDirectory ${knownDirectory}") + if (knownDirectory != null) return knownDirectory.booleanValue() + + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + if (!path) return true // consider root a directory + + // how to determine? not exists but has files in it + if (s3Client.doesObjectExist(bucketName, path)) { + knownDirectory = Boolean.FALSE + return false + } + + ListObjectsV2Request lor = new ListObjectsV2Request().withBucketName(bucketName).withPrefix(path).withDelimiter("/").withMaxKeys(1) + ListObjectsV2Result result = s3Client.listObjectsV2(lor) + if (result.getObjectSummaries() || result.getCommonPrefixes()) { + knownDirectory = Boolean.TRUE + return true + } else { + return false + } } @Override List getDirectoryEntries() { - // TODO - return null + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + // logger.warn("getDirectoryEntries bucket ${bucketName} path ${path}") + + ListObjectsV2Request lor = new ListObjectsV2Request().withBucketName(bucketName).withPrefix(path + "/").withDelimiter("/") + ListObjectsV2Result result = s3Client.listObjectsV2(lor) + // common prefixes (sub-directories) + List commonPrefixList = result.getCommonPrefixes() + // objects (files in directory) + List objectList = result.getObjectSummaries() + // add to the list + ArrayList entryList = new ArrayList<>(commonPrefixList.size() + objectList.size()) + for (String subDir in commonPrefixList) + entryList.add(new S3ResourceReference().init(location + '/' + subDir, Boolean.TRUE, ecf)) + for (S3ObjectSummary os in objectList) + entryList.add(new S3ResourceReference().init(locationPrefix + os.getBucketName() + '/' + os.getKey(), Boolean.FALSE, ecf)) + // logger.warn("sub-dirs: ${commonPrefixList.join(', ')}") + // logger.warn("files: ${objectList.collect({it.getKey()}).join(', ')}") + // logger.warn("RR entries: ${entryList.collect({it.toString()}).join(', ')}") + return entryList } @Override boolean supportsExists() { true } - @Override boolean getExists() { return getDbResource(true) != null } + @Override boolean getExists() { + if (knownDirectory != null && knownDirectory.booleanValue()) return true + + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + // first see if it's a file + boolean existingFile = s3Client.doesObjectExist(bucketName, path) + if (existingFile) { + knownDirectory = Boolean.FALSE // known file + return true + } + + // handle directories by seeing if is a prefix with any files, limit 1 for efficiency + ListObjectsV2Request lor = new ListObjectsV2Request().withBucketName(bucketName).withPrefix(path).withDelimiter("/").withMaxKeys(1) + ListObjectsV2Result result = s3Client.listObjectsV2(lor) + if (result.getObjectSummaries() || result.getCommonPrefixes()) { + knownDirectory = Boolean.TRUE + return true + } else { + return false + } + } @Override boolean supportsLastModified() { true } @Override long getLastModified() { - // TODO - return 0 + ObjectMetadata om = s3Client.getObjectMetadata(getBucketName(location), getPath(location)) + if (om == null) return 0 + return om.getLastModified()?.getTime() } @Override boolean supportsSize() { true } @Override long getSize() { - // TODO - return 0 + ObjectMetadata om = s3Client.getObjectMetadata(getBucketName(location), getPath(location)) + if (om == null) return 0 + // NOTE: or use getInstanceLength()? + return om.getContentLength() } @Override boolean supportsWrite() { true } @Override void putText(String text) { - // TODO: use diff from last version for text - SerialBlob sblob = text ? new SerialBlob(text.getBytes(StandardCharsets.UTF_8)) : null - // TODO + // FUTURE: use diff from last version for text + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + if (autoCreateBucket && !s3Client.doesBucketExistV2(bucketName)) s3Client.createBucket(bucketName) + + s3Client.putObject(bucketName, path, text) } @Override void putStream(InputStream stream) { if (stream == null) return - ByteArrayOutputStream baos = new ByteArrayOutputStream() - ObjectUtilities.copyStream(stream, baos) - SerialBlob sblob = new SerialBlob(baos.toByteArray()) - // TODO + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + if (autoCreateBucket && !s3Client.doesBucketExistV2(bucketName)) s3Client.createBucket(bucketName) + + // NOTE: can specify more options using ObjectMetadata object as 4th parameter + // NOTE: return PutObjectResult with more info, including version/etc + // FUTURE: somehow ObjectMetadata.setContentLength()? without that will locally buffer entire stream to calculate length, ie Content-Length HTTP header required for REST API + s3Client.putObject(bucketName, path, stream, null) } @Override void move(String newLocation) { @@ -134,56 +283,127 @@ class S3ResourceReference extends BaseResourceReference { if (!newLocation.startsWith(locationPrefix)) throw new BaseArtifactException("Location [${newLocation}] is not a s3 location, not moving resource at ${getLocation()}") - List filenameList = new ArrayList<>(Arrays.asList(newLocation.substring(locationPrefix.length()).split("/"))) - if (filenameList) { - String newFilename = filenameList.get(filenameList.size()-1) - filenameList.remove(filenameList.size()-1) - // TODO - } + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + String newBucketName = getBucketName(newLocation) + String newPath = getPath(newLocation) + + if (autoCreateBucket && bucketName != newBucketName && !s3Client.doesBucketExistV2(newBucketName)) s3Client.createBucket(newBucketName) + + // FUTURE: handle source version somehow, maybe different move or copy method? pass as third parameter to CopyObjectRequest constructor + CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucketName, path, newBucketName, newPath) + s3Client.copyObject(copyObjRequest) + s3Client.deleteObject(bucketName, path) } @Override ResourceReference makeDirectory(String name) { - // TODO can make directory with no files in S3? + // NOTE can make directory with no files in S3? seems no, directory is just a partial object key return new S3ResourceReference().init("${location}/${name}", ecf) } @Override ResourceReference makeFile(String name) { - S3ResourceReference newRef = new S3ResourceReference().init("${location}/${name}", ecf) - // TODO make empty file + S3ResourceReference newRef = new S3ResourceReference() + newRef.init("${location}/${name}", ecf) + // TODO make empty file? return newRef } @Override boolean delete() { - // TODO - // if not exists: return false + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + if (!s3Client.doesObjectExist(bucketName, path)) return false + s3Client.deleteObject(bucketName, path) return true } @Override boolean supportsVersion() { return true } @Override Version getVersion(String versionName) { - // TODO - return null + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + GetObjectMetadataRequest gomr = new GetObjectMetadataRequest(bucketName, path, versionName) + ObjectMetadata om = s3Client.getObjectMetadata(gomr) + // TODO: use setUserMetadata(Map userMetadata) and getUserMetadata() for userId, needs to be on app puts/etc + // TODO: worth a separate request to try to get previousVersionName? doesn't seem to be easy way to do that either... + return new Version(this, om.getVersionId(), null, null, new Timestamp(om.getLastModified().getTime())) } @Override Version getCurrentVersion() { - // TODO - return null + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + try { + ObjectMetadata om = s3Client.getObjectMetadata(bucketName, path) + String versionName = om.getVersionId() + if (versionName == null || versionName.isEmpty()) return null + // TODO: use setUserMetadata(Map userMetadata) and getUserMetadata() for userId, needs to be on app puts/etc + // TODO: worth a separate request to try to get previousVersionName? doesn't seem to be easy way to do that either... + return new Version(this, versionName, null, null, new Timestamp(om.getLastModified().getTime())) + } catch (AmazonS3Exception e) { + if (e.getStatusCode() == 404) { + logger.warn("Not found (404) error in get version for bucket ${bucketName} path ${path}: ${e.toString()}") + return null + } else { throw e } + } } @Override Version getRootVersion() { - // TODO - return null + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + // NOTE: assuming this does oldest first, needs testing, docs not clear on any of this stuff + ListVersionsRequest lvr = new ListVersionsRequest().withBucketName(bucketName).withPrefix(path).withMaxResults(1) + VersionListing vl = s3Client.listVersions(lvr) + List s3vsList = vl.getVersionSummaries() + if (s3vsList == null || s3vsList.size() == 0) return null + S3VersionSummary s3vs = s3vsList.get(0) + return new Version(this, s3vs.getVersionId(), null, null, new Timestamp(s3vs.getLastModified().getTime())) } @Override ArrayList getVersionHistory() { - // TODO - ArrayList verList = new ArrayList<>() - return verList + return getNextVersions(null) } @Override ArrayList getNextVersions(String versionName) { - // TODO - ArrayList verList = new ArrayList<>() + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + // NOTE: any way to get versions that have this versionName as the previous version? doesn't seem so, ie no branching just linear list so just get next version + ListVersionsRequest lvr = new ListVersionsRequest().withBucketName(bucketName).withPrefix(path).withMaxResults(1) + if (versionName != null && !versionName.isEmpty()) lvr.withVersionIdMarker(versionName) + VersionListing vl = s3Client.listVersions(lvr) + List s3vsList = vl.getVersionSummaries() + ArrayList verList = new ArrayList<>(s3vsList.size()) + for (S3VersionSummary s3vs in s3vsList) + verList.add(new Version(this, s3vs.getVersionId(), null, null, new Timestamp(s3vs.getLastModified().getTime()))) return verList } @Override InputStream openStream(String versionName) { if (versionName == null || versionName.isEmpty()) return openStream() - // TODO - return null + + AmazonS3 s3Client = getS3Client() + String bucketName = getBucketName(location) + String path = getPath(location) + + try { + GetObjectRequest gor = new GetObjectRequest(bucketName, path, versionName) + S3Object obj = s3Client.getObject(gor) + S3ObjectInputStream s3is = obj.getObjectContent() + return s3is + } catch (AmazonS3Exception e) { + if (e.getStatusCode() == 404) { + logger.warn("Not found (404) error in openStream for bucket ${bucketName} path ${path} version ${versionName}: ${e.toString()}") + return null + } else { throw e } + } } @Override String getText(String versionName) { return ObjectUtilities.getStreamText(openStream(versionName)) } + + AmazonS3 getS3Client() { + AmazonS3 s3Client = ecf.getTool(S3ClientToolFactory.TOOL_NAME, AmazonS3.class) + if (s3Client == null) throw new BaseException("AWS S3 Client not initialized") + return s3Client + } }