Support directories in PUT/GET

Author: sfc-gh-ext-simba-hx

cpp/FileMetadataInitializer.cpp

@@ -8,6 +8,7 @@
 #include "snowflake/platform.h"
 #include "snowflake/SnowflakeTransferException.hpp"
 #include <cerrno>
+#include "boost/filesystem.hpp"
 
 #define COMPRESSION_AUTO "AUTO"
 #define COMPRESSION_AUTO_DETECT "AUTO_DETECT"
@@ -20,7 +21,7 @@
 #include <fnmatch.h>
 #endif
 
-
+using namespace boost::filesystem;
 
 Snowflake::Client::FileMetadataInitializer::FileMetadataInitializer(
   std::vector<FileMetadata> &smallFileMetadata,
@@ -34,16 +35,16 @@ Snowflake::Client::FileMetadataInitializer::FileMetadataInitializer(
 }
 
 void
-Snowflake::Client::FileMetadataInitializer::initUploadFileMetadata(const std::string &fileDir, const char *fileName,
+Snowflake::Client::FileMetadataInitializer::initUploadFileMetadata(const std::string &fileNameFull,
+                                                                   const std::string &destPath,
+                                                                   const std::string &fileName,
                                                                    size_t fileSize, size_t threshold)
 {
-  std::string fileNameFull = fileDir;
-  fileNameFull += fileName;
-
   FileMetadata fileMetadata;
   fileMetadata.srcFileName = m_stmtPutGet->platformStringToUTF8(fileNameFull);
   fileMetadata.srcFileSize = fileSize;
-  fileMetadata.destFileName = m_stmtPutGet->platformStringToUTF8(std::string(fileName));
+  fileMetadata.destPath = m_stmtPutGet->platformStringToUTF8(destPath);
+  fileMetadata.destFileName = m_stmtPutGet->platformStringToUTF8(fileName);
   // process compression type
   initCompressionMetadata(fileMetadata);
 
@@ -56,9 +57,52 @@ Snowflake::Client::FileMetadataInitializer::initUploadFileMetadata(const std::st
 
 void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(std::string &sourceLocation,
                                                                               size_t putThreshold)
+{
+  // looking for files on disk.
+  std::string srcLocationPlatform = m_stmtPutGet->UTF8ToPlatformString(sourceLocation);
+  size_t dirSep = srcLocationPlatform.find_last_of(PATH_SEP);
+  std::string basePath = srcLocationPlatform.substr(0, dirSep + 1);
+
+  std::vector<std::string> fileList;
+  if (!listFilesRecursive(srcLocationPlatform, fileList))
+  {
+    CXX_LOG_ERROR("Failed on finding files for uploading.");
+    return;
+  }
+
+  for (auto file = fileList.begin(); file != fileList.end(); file++)
+  {
+    path p(*file);
+    size_t fileSize = file_size(p);
+    std::string fileNameFull = p.string();
+    std::string fileName = p.filename().string();
+    //make the path on stage by removing base path and file name from full path
+    std::string destPath = fileNameFull.substr(basePath.length(),
+                                               fileNameFull.length() - basePath.length() - fileName.length());
+    initUploadFileMetadata(fileNameFull, destPath, fileName, fileSize, putThreshold);
+  }
+}
+
+void Snowflake::Client::FileMetadataInitializer::includeSubfolderFilesRecursive(const std::string &folderPath,
+                                                                                std::vector<std::string> & fileList)
+{
+  for (auto const& entry : recursive_directory_iterator(folderPath))
+  {
+    if (is_regular_file(entry))
+    {
+      fileList.push_back(entry.path().string());
+    }
+  }
+}
+
+bool Snowflake::Client::FileMetadataInitializer::listFilesRecursive(const std::string &sourceLocation,
+                                                                    std::vector<std::string> & fileList)
 {
 // looking for files on disk. 
   std::string srcLocationPlatform = m_stmtPutGet->UTF8ToPlatformString(sourceLocation);
+  size_t dirSep = srcLocationPlatform.find_last_of(PATH_SEP);
+  std::string dirPath = srcLocationPlatform.substr(0, dirSep + 1);
+  std::string filePattern = srcLocationPlatform.substr(dirSep + 1);
 
 #ifdef _WIN32
   WIN32_FIND_DATA fdd;
@@ -71,8 +115,7 @@ void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(st
     {
       CXX_LOG_ERROR("No file matching pattern %s has been found. Error: %d", 
         sourceLocation.c_str(), dwError);
-      FindClose(hFind);
-      return;
+      return false;
     }
     else if (dwError != ERROR_SUCCESS)
     {
@@ -85,37 +128,24 @@ void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(st
   do {
     if (!(fdd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) )
     {
-      std::string fileFullPath = std::string(fdd.cFileName);
-      size_t dirSep = srcLocationPlatform.find_last_of(PATH_SEP);
-      if (dirSep == std::string::npos) 
-      {
-        dirSep = sourceLocation.find_last_of(ALTER_PATH_SEP);
-      }
-      if (dirSep != std::string::npos) 
-      {
-        std::string dirPath = srcLocationPlatform.substr(0, dirSep + 1);
-        LARGE_INTEGER fileSize;
-        fileSize.LowPart = fdd.nFileSizeLow;
-        fileSize.HighPart = fdd.nFileSizeHigh;
-        initUploadFileMetadata(dirPath, (char *)fdd.cFileName, (size_t)fileSize.QuadPart, putThreshold);
-      }
+      fileList.push_back(dirPath + fdd.cFileName);
+    }
+    else
+    {
+      includeSubfolderFilesRecursive(dirPath + fdd.cFileName, fileList);
     }
   } while (FindNextFile(hFind, &fdd) != 0);
 
   DWORD dwError = GetLastError();
+  FindClose(hFind);
   if (dwError != ERROR_NO_MORE_FILES)
   {
     CXX_LOG_ERROR("Failed on FindNextFile. Error: %d", dwError);
     throw SnowflakeTransferException(TransferError::DIR_OPEN_ERROR,
       srcLocationPlatform.c_str(), dwError);
   }
-  FindClose(hFind);
 
 #else
-  unsigned long dirSep = srcLocationPlatform.find_last_of(PATH_SEP);
-  std::string dirPath = srcLocationPlatform.substr(0, dirSep + 1);
-  std::string filePattern = srcLocationPlatform.substr(dirSep + 1);
-
   DIR * dir = nullptr;
   struct dirent * dir_entry;
   if ((dir = opendir(dirPath.c_str())) != NULL)
@@ -130,8 +160,11 @@ void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(st
         if (!ret)
         {
           if (S_ISREG(fileStatus.st_mode)) {
-            initUploadFileMetadata(dirPath, dir_entry->d_name,
-                                   (size_t) fileStatus.st_size, putThreshold);
+            fileList.pushback(dirPath + dir_entry->d_name);
+          }
+          else if (S_ISDIR(fileStatus.st_mode))
+          {
+            includeSubfolderFilesRecursive(dirPath + dir_entry->d_name, fileList);
           }
         }
         else
@@ -169,7 +202,7 @@ void Snowflake::Client::FileMetadataInitializer::initCompressionMetadata(
     // guess
     CXX_LOG_INFO("Auto detect on compression type");
     fileMetadata.sourceCompression = FileCompressionType::guessCompressionType(
-      fileMetadata.srcFileName);
+      m_stmtPutGet->UTF8ToPlatformString(fileMetadata.srcFileName));
   }
   else if (!sf_strncasecmp(m_sourceCompression, COMPRESSION_NONE, 
                         sizeof(COMPRESSION_NONE)))
@@ -253,8 +286,9 @@ populateSrcLocDownloadMetadata(std::string &sourceLocation,
                                size_t getThreshold)
 {
   std::string fullPath = *remoteLocation + sourceLocation;
-  size_t dirSep = fullPath.find_last_of('/');
-  std::string dstFileName = fullPath.substr(dirSep + 1);
+  size_t dirSep = sourceLocation.find_last_of('/');
+  std::string dstFileName = sourceLocation.substr(dirSep + 1);
+  std::string dstPath = sourceLocation.substr(0, dirSep + 1);
 
   FileMetadata fileMetadata;
   fileMetadata.presignedUrl = presignedUrl;
@@ -271,6 +305,7 @@ populateSrcLocDownloadMetadata(std::string &sourceLocation,
     metaListToPush.push_back(fileMetadata);
     metaListToPush.back().srcFileName = fullPath;
     metaListToPush.back().destFileName = dstFileName;
+    metaListToPush.back().destPath = dstPath;
     if (encMat)
     {
       EncryptionProvider::decryptFileKey(&(metaListToPush.back()), encMat, getRandomDev());

cpp/FileMetadataInitializer.hpp

@@ -30,11 +30,37 @@ class FileMetadataInitializer
                           IStatementPutGet *stmtPutGet);
 
   /**
-   * Given a source locations, find all files that match the location pattern,
+   * Given a source location, find all files that match the location pattern,
    * init file metadata, and divide them into different vector according to size
    */
   void populateSrcLocUploadMetadata(std::string &sourceLocation, size_t putThreshold);
 
+
+  /**
+  * Utility function to replace all matching instances in a string.
+  */
+  static void replaceStrAll(std::string& stringToReplace, std::string const& oldValue,
+                            std::string const& newValue);
+  /**
+  * Given a source location, find all files recursively in subfolders that match the
+  * location pattern. Utility function called from populateSrcLocUploadMetadata.
+  *
+  * @param sourceLocation The source location could have pattern at the end.
+  * @param fileList Output the files with the full path.
+  *
+  * @return True when succeeded, false when no file matches with the source location.
+  * @throw SnowflakeTransferException on unexpected error.
+  */
+  bool listFilesRecursive(const std::string &sourceLocation, std::vector<std::string> & fileList);
+
+  /**
+  * Given a full path of a folder, add all files in the folder recursively including subfolders.
+  *
+  * @param folderPath The full path of a folder.
+  * @param fileList Output the files in the folder recursively including subfolders.
+  */
+  void includeSubfolderFilesRecursive(const std::string &folderPath, std::vector<std::string> & fileList);
+
   /**
    * Given a source location, find out file size to determine use parallel
    * download or not.
@@ -79,7 +105,8 @@ class FileMetadataInitializer
    * Given file name, populate metadata
    * @param fileName
    */
-  void initUploadFileMetadata(const std::string &fileDir, const char *fileName, size_t fileSize, size_t threshold);
+  void initUploadFileMetadata(const std::string &fileNameFull, const std::string &destPath,
+                              const std::string &fileName, size_t fileSize, size_t threshold);
 
   /**
    * init compression metadata

cpp/FileTransferAgent.cpp

@@ -468,6 +468,12 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::uploadSingleFi
     fileMetadata->srcFileToUpload = fileMetadata->srcFileName;
     fileMetadata->srcFileToUploadSize = fileMetadata->srcFileSize;
   }
+
+  // after compress replace PATH_SEP with / in destPath as that's
+  // what needed on stage side
+  replaceStrAll(fileMetadata->destPath, std::string() + PATH_SEP, "/");
+  fileMetadata->destFileName = fileMetadata->destPath + fileMetadata->destFileName;
+
   CXX_LOG_TRACE("Update File digest metadata start");
 
   // calculate digest
@@ -614,6 +620,24 @@ void Snowflake::Client::FileTransferAgent::compressSourceFile(
   }
 
   std::string stagingFile(tempDir);
+
+  if (!fileMetadata->destPath.empty())
+  {
+    std::string subfolder = fileMetadata->destPath;
+    replaceStrAll(subfolder, "/", std::string() + PATH_SEP);
+    std::string subfolderPlatform = m_stmtPutGet->UTF8ToPlatformString(subfolder);
+    subfolder = std::string(tempDir) + subfolder;
+    subfolderPlatform = std::string(tempDir) + subfolderPlatform;
+
+    int ret = sf_create_directory_if_not_exists_recursive(subfolderPlatform.c_str());
+    if (ret != 0)
+    {
+      CXX_LOG_ERROR("Failed to create temporary folder %s. Errno: %d", subfolder, errno);
+      throw SnowflakeTransferException(TransferError::FILE_OPEN_ERROR, subfolder, -1);
+    }
+    stagingFile = subfolderPlatform;
+  }
+
   stagingFile += m_stmtPutGet->UTF8ToPlatformString(fileMetadata->destFileName);
   std::string srcFileNamePlatform = m_stmtPutGet->UTF8ToPlatformString(fileMetadata->srcFileName);
 
@@ -647,12 +671,13 @@ void Snowflake::Client::FileTransferAgent::download(string *command)
   m_executionResults = new FileTransferExecutionResult(CommandType::DOWNLOAD,
     m_largeFilesMeta.size() + m_smallFilesMeta.size());
 
-  int ret = sf_create_directory_if_not_exists((const char *)response.localLocation);
+  std::string localLocationPlatform = m_stmtPutGet->UTF8ToPlatformString(response.localLocation);
+  int ret = sf_create_directory_if_not_exists_recursive((const char *)localLocationPlatform.c_str());
   if (ret != 0)
   {
     CXX_LOG_ERROR("Filed to create directory %s", response.localLocation);
     throw SnowflakeTransferException(TransferError::MKDIR_ERROR, 
-      response.localLocation, ret);
+      localLocationPlatform, ret);
   }
 
   if (m_largeFilesMeta.size() > 0)
@@ -829,11 +854,28 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::downloadSingle
   FileMetadata *fileMetadata,
   size_t resultIndex)
 {
+   RemoteStorageRequestOutcome outcome = RemoteStorageRequestOutcome::FAILED;
+
+   // create subfolder first befor adding file name
+   replaceStrAll(fileMetadata->destPath, "/", std::string() + PATH_SEP);
    fileMetadata->destPath = std::string(response.localLocation) + PATH_SEP +
-    fileMetadata->destFileName;
+     fileMetadata->destPath;
    std::string destPathPlatform = m_stmtPutGet->UTF8ToPlatformString(fileMetadata->destPath);
+   int ret = sf_create_directory_if_not_exists_recursive(destPathPlatform.c_str());
+   if (ret != 0)
+   {
+     char* str_error = sf_strerror(errno);
+     CXX_LOG_DEBUG("Filed to create directory: %s",
+                   fileMetadata->destPath.c_str(), str_error);
+     sf_free_s(str_error);
+     m_executionResults->SetTransferOutCome(outcome, resultIndex);
+
+     return outcome;
+   }
+
+   fileMetadata->destPath += fileMetadata->destFileName;
+   destPathPlatform += m_stmtPutGet->UTF8ToPlatformString(fileMetadata->destFileName);
 
-   RemoteStorageRequestOutcome outcome = RemoteStorageRequestOutcome::FAILED;
    RetryContext getRetryCtx(fileMetadata->srcFileName, m_maxGetRetries);
    do
    {
@@ -846,7 +888,6 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::downloadSingle
                                           std::ios_base::out | std::ios_base::binary);
      }
      catch (...) {
-       std::string err = "Could not open file " + fileMetadata->destPath + " to downoad";
        char* str_error = sf_strerror(errno);
        CXX_LOG_DEBUG("Could not open file %s to downoad: %s",
                      fileMetadata->destPath.c_str(), str_error);
@@ -856,7 +897,6 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::downloadSingle
      }
      if (!dstFile.is_open())
      {
-       std::string err = "Could not open file " + fileMetadata->destPath + " to downoad";
        char* str_error = sf_strerror(errno);
        CXX_LOG_DEBUG("Could not open file %s to downoad: %s",
                      fileMetadata->destPath.c_str(), str_error);