Support Pytorch & Tensorflow Formats on Datasets (#4)

* feat: add explore()

* fix: increase limits for ids fetching

* feat: add parallel fetch

* add pypi publishing

Co-authored-by: Pablo <[email protected]>

Author: PJEstrada

.github/workflows/publish_pypi.yaml

@@ -0,0 +1,34 @@
+name: Publish Docker Images
+on:
+  release:
+    types: [published]
+jobs:
+  Publish-SDK-pypi:
+    runs-on: ubuntu-latest
+    env:
+      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+    steps:
+    - name: Check out repository code
+      uses: actions/checkout@v2
+    - name: Get Release Tag
+      run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+    - name: Test
+      run: |
+        echo $RELEASE_VERSION
+        echo ${{ env.RELEASE_VERSION }}
+    - name: Set Release to Diffgram SDK version
+      run: |
+        export DIFFGRAM_SDK_VERSION=${{ env.RELEASE_VERSION }}
+    - name: Echo
+      run: echo ${{ steps.tag.outputs.result }}
+    - name: Install Twine
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y twine
+    - name: Build Dist
+      run: |
+        cd sdk
+        python setup.py sdist bdist_wheel
+    - name: Upload with Twine
+      run: twine upload dist/* --skip-existing --username ${{ secrets.TWINE_PASSWORD }} --password ${{ secrets.TWINE_USERNAME }}

sdk/diffgram/__init__.py

@@ -1,5 +1,6 @@
+import os
 __name__ = "diffgram"
-__version__ = "0.3.4"
+__version__ = os.getenv('DIFFGRAM_SDK_VERSION')
 
 from diffgram.core.core import Project
 from diffgram.file.file import File

sdk/diffgram/core/core.py

@@ -49,20 +49,18 @@ def __init__(
 			self.host = host
 		self.directory_id = None
 		self.name_to_file_id = None
-
 		self.auth(
 			project_string_id = project_string_id,
 			client_id = client_id, 
 			client_secret = client_secret)
-
 		self.file = FileConstructor(self)
 		self.train = Train(self)
 		self.job = Job(self)
 		self.guide = Guide(self)
-		self.directory = Directory(self)
+		self.directory = Directory(self, validate_ids = False)
 		self.export = Export(self)
 		self.task = Task(client = self)
-
+		
 	def get_member_list(self):
 		url = '/api/project/{}/view'.format(self.project_string_id)
 		response = self.session.get(url=self.host + url)

sdk/diffgram/core/diffgram_dataset_iterator.py

@@ -4,7 +4,7 @@
 
 class DiffgramDatasetIterator:
 
-    def __init__(self, project, diffgram_file_id_list):
+    def __init__(self, project, diffgram_file_id_list, validate_ids = True):
         """
 
         :param project (sdk.core.core.Project): A Project object from the Diffgram SDK
@@ -14,7 +14,8 @@ def __init__(self, project, diffgram_file_id_list):
 
         self.project = project
         self._internal_file_list = []
-        self.__validate_file_ids()
+        if validate_ids:
+            self.__validate_file_ids()
         self.current_file_index = 0
 
     def __iter__(self):
@@ -37,6 +38,8 @@ def __next__(self):
         return instance_data
 
     def __validate_file_ids(self):
+        if not self.diffgram_file_id_list:
+            return
         result = self.project.file.file_list_exists(self.diffgram_file_id_list)
         if not result:
             raise Exception(