feat(CLI): Introduce command line interface for uploads/downloads

README.md

@@ -24,12 +24,38 @@ $ git clone https://github.com/frameio/python-frameio-client
 $ pip install .
 ```
 
-_Note: The Frame.io Python client may not work correctly in Python 3.8+_
+### Developing
+Install the package into your development environment and link to it by running the following:
+
+```sh
+pipenv install -e . -pre
+```
 
 ## Documentation
 
 [Frame.io API Documentation](https://developer.frame.io/docs)
 
+### Use CLI
+When you install this package, a cli tool called `fioctl` will also be installed to your environment.
+
+**To upload a file or folder**
+```sh
+fioctl \
+--token fio-u-YOUR_TOKEN_HERE  \
+--destination "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \
+--target "YOUR LOCAL SYSTEM DIRECTORY" \
+--threads 8
+```
+
+**To download a file, project, or folder**
+```sh
+fioctl \
+--token fio-u-YOUR_TOKEN_HERE  \
+--destination "YOUR LOCAL SYSTEM DIRECTORY" \
+--target "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \
+--threads 2
+```
+
 ## Usage
 
 _Note: A valid token is required to make requests to Frame.io. Go to our [Developer Portal](https://developer.frame.io/) to get a token!_

examples/recursive_upload.py

@@ -0,0 +1,94 @@
+import os
+import time
+import mimetypes
+import concurrent.futures
+import threading
+from frameioclient import FrameioClient
+from pprint import pprint
+
+global file_num
+file_num = 0
+
+global file_count
+file_count = 0
+
+def create_n_upload(task):
+    client=task[0]
+    file_p=task[1]
+    parent_asset_id=task[2]
+    abs_path = os.path.abspath(file_p)
+    file_s = os.path.getsize(file_p)
+    file_n = os.path.split(file_p)[1]
+    file_mime = mimetypes.guess_type(abs_path)[0]
+
+    asset = client.create_asset(
+      parent_asset_id=parent_asset_id,  
+      name=file_n,
+      type="file",
+      filetype=file_mime,
+      filesize=file_s
+    )
+
+    with open(abs_path, "rb") as ul_file:
+        asset_info = client.upload(asset, ul_file)
+
+    return asset_info
+
+
+def create_folder(folder_n, parent_asset_id):
+    asset = client.create_asset(
+      parent_asset_id=parent_asset_id,  
+      name=folder_n,
+      type="folder",
+    )
+
+    return asset['id']
+
+
+def file_counter(root_folder):
+    matches = []
+    for root, dirnames, filenames in os.walk(root_folder):
+        for filename in filenames:
+            matches.append(os.path.join(filename))
+
+    return matches
+
+
+def recursive_upload(client, folder, parent_asset_id):
+    # Seperate files and folders:
+    file_list = list()
+    folder_list = list()
+
+    for item in os.listdir(folder):
+        if item == ".DS_Store": # Ignore .DS_Store files on Mac
+            continue
+
+        complete_item_path = os.path.join(folder, item)
+
+        if os.path.isfile(complete_item_path):
+            file_list.append(item)
+        else:
+            folder_list.append(item)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        for file_p in file_list:
+            global file_num
+            file_num += 1
+            print(f"Starting {file_num}/{file_count}")
+            complete_dir_obj = os.path.join(folder, file_p)
+            task = (client, complete_dir_obj, parent_asset_id)
+            executor.submit(create_n_upload, task)
+
+    for folder_i in folder_list:
+        new_folder = os.path.join(folder, folder_i)
+        new_parent_asset_id = create_folder(folder_i, parent_asset_id)
+        recursive_upload(client, new_folder, new_parent_asset_id)
+
+
+if __name__ == "__main__":
+    root_folder = "./test_structure"
+    parent_asset_id = "PARENT_ASSET_ID"
+    client = FrameioClient(os.getenv("FRAME_IO_TOKEN"))
+
+    file_count = len(file_counter(root_folder))
+    recursive_upload(client, root_folder, parent_asset_id)

frameioclient/client.py

@@ -8,8 +8,8 @@
 )
 
 class FrameioClient(APIClient, object):
-  def __init__(self, token, host='https://api.frame.io'):
-    super().__init__(token, host)
+  def __init__(self, token, host='https://api.frame.io', threads=5, progress=False):
+    super().__init__(token, host, threads, progress)
 
   @property
   def me(self):

frameioclient/fiocli.py

@@ -0,0 +1,52 @@
+import os
+import sys
+import argparse
+
+from frameioclient import FrameioClient
+
+
+def main():
+    parser=argparse.ArgumentParser(prog='fiocli', description='Frame.io Python SDK CLI')
+
+    ## Define args
+    parser.add_argument('--token', action='store', metavar='token', type=str, nargs='+', help='Developer Token')
+    # parser.add_argument('--op', action='store', metavar='op', type=str, nargs='+', help='Operation: upload, download')
+    parser.add_argument('--target', action='store', metavar='target', type=str, nargs='+', help='Target: remote project or folder, or alternatively a local file/folder')
+    parser.add_argument('--destination', action='store', metavar='destination', type=str, nargs='+', help='Destination: remote project or folder, or alternatively a local file/folder')
+    parser.add_argument('--threads', action='store', metavar='threads', type=int, nargs='+', help='Number of threads to use')
+
+    ## Parse args
+    args = parser.parse_args()
+
+    if args.threads:
+        threads = args.threads[0]
+    else:
+        threads = 5
+
+    ## Handle args
+    if args.token:
+        client = None
+        # print(args.token)
+        try:
+            client = FrameioClient(args.token[0], progress=True, threads=threads)
+        except Exception as e:
+            print("Failed")
+            sys.exit(1)
+
+        # If args.op == 'upload':
+        if args.target:
+            if args.destination:
+                # Check to see if this is a local target and thus a download
+                if os.path.isdir(args.destination[0]):
+                    asset = client.assets.get(args.target[0])
+                    return client.assets.download(asset, args.destination[0], progress=True, multi_part=True, concurrency=threads)
+                else: # This is an upload
+                    if os.path.isdir(args.target[0]):
+                        return client.assets.upload_folder(args.target[0], args.destination[0])
+                    else:
+                        return client.assets.upload(args.destination[0], args.target[0])
+            else:
+                print("No destination supplied")
+        else:
+            print("No target supplied")
+

frameioclient/lib/download.py

@@ -35,6 +35,7 @@ def __init__(self, asset, download_folder, prefix, multi_part=False, replace=Fal
     self.futures = list()
     self.checksum = None
     self.original_checksum = None
+    self.checksum_verification = True
     self.chunk_size = (25 * 1024 * 1024) # 25 MB chunk size
     self.chunks = math.ceil(self.file_size/self.chunk_size)
     self.prefix = prefix
@@ -45,6 +46,7 @@ def __init__(self, asset, download_folder, prefix, multi_part=False, replace=Fal
     self.session = self.aws_client._get_session(auth=None)
     self.filename = Utils.normalize_filename(asset["name"])
     self.request_logs = list()
+    self.stats = True
 
     self._evaluate_asset()
     self._get_path()
@@ -142,32 +144,34 @@ def download_handler(self):
       print("Destination folder not found, creating")
       os.mkdir(self.download_folder)
 
-    if not self.replace:
-      if os.path.isfile(self.get_path()):
-        print("File already exists at this location.")
-        return self.destination
-    else:
-      url = self.get_download_key()
+    if os.path.isfile(self.get_path()) == False:
+      pass
 
-      if self.watermarked == True:
-        return self.single_part_download(url)
+    if os.path.isfile(self.get_path()) and self.replace == True:
+      os.remove(self.get_path())
+
+    if os.path.isfile(self.get_path()) and self.replace == False:
+      print("File already exists at this location.")
+      return self.destination
+
+    url = self.get_download_key()
+
+    if self.watermarked == True:
+      return self.single_part_download(url)
+    else:
+      # Don't use multi-part download for files below 25 MB
+      if self.asset['filesize'] < 26214400:
+        return self.download(url)
+      if self.multi_part == True:
+        return self.multi_part_download(url)
       else:
-        # Don't use multi-part download for files below 25 MB
-        if self.asset['filesize'] < 26214400:
-          return self.download(url)
-        if self.multi_part == True:
-          return self.multi_part_download(url)
-        else:
-          return self.single_part_download(url)
+        return self.single_part_download(url)
 
   def single_part_download(self, url):
     start_time = time.time()
     print("Beginning download -- {} -- {}".format(self.asset["name"], Utils.format_bytes(self.file_size, type="size")))
 
     # Downloading
-    r = self.session.get(url)
-    open(self.destination, "wb").write(r.content)
-
     with open(self.destination, 'wb') as handle:
       try:
         # TODO make sure this approach works for SBWM download
@@ -295,7 +299,7 @@ def multi_part_download(self, url):
       # Submit telemetry
       transfer_stats = {'speed': download_speed, 'time': download_time, 'cdn': AWSClient.check_cdn(url)}
 
-      Event(self.user_id, 'python-sdk-download-stats', transfer_stats)
+      # Event(self.user_id, 'python-sdk-download-stats', transfer_stats)
 
     # If stats = True, we return a dict with way more info, otherwise \
     if self.stats:
@@ -305,7 +309,7 @@ def multi_part_download(self, url):
         "speed": download_speed,
         "elapsed": download_time,
         "cdn": AWSClient.check_cdn(url),
-        "concurrency": self.concurrency,
+        "concurrency": self.aws_client.concurrency,
         "size": self.file_size,
         "chunks": self.chunks
       }

frameioclient/lib/transport.py

@@ -46,24 +46,27 @@ def _get_session(self, auth=True):
 
 
 class APIClient(HTTPClient, object):
-    def __init__(self, token, host):
+    def __init__(self, token, host, threads, progress):
         super().__init__()
         self.host = host
         self.token = token
+        self.threads = threads
+        self.progress = progress
         self._initialize_thread()
-        self.session = self._get_session(auth=token)
+        self.session = self._get_session()
         self.auth_header = {
-            'Authorization': 'Bearer {}'.format(self.token),
+            'Authorization': 'Bearer {}'.format(self.token)
         }
 
-    def _api_call(self, method, endpoint, payload={}, limit=None):
-        url = '{}/v2{}'.format(self.host, endpoint)
+    def _format_api_call(self, endpoint):
+        return '{}/v2{}'.format(self.host, endpoint)
 
+    def _api_call(self, method, endpoint, payload={}, limit=None):
         headers = {**self.shared_headers, **self.auth_header}
 
         r = self.session.request(
             method,
-            url,
+            self._format_api_call(endpoint),
             headers=headers,
             json=payload
         )
@@ -98,14 +101,14 @@ def get_specific_page(self, method, endpoint, payload, page):
         Gets a specific page for that endpoint, used by Pagination Class
 
         :Args:
-        method (string): 'get', 'post'
-        endpoint (string): endpoint ('/accounts/<ACCOUNT_ID>/teams')
-        payload (dict): Request payload
-        page (int): What page to get
+            method (string): 'get', 'post'
+            endpoint (string): endpoint ('/accounts/<ACCOUNT_ID>/teams')
+            payload (dict): Request payload
+            page (int): What page to get
         """
         if method == 'get':
             endpoint = '{}?page={}'.format(endpoint, page)
-        return self._api_call(method, endpoint)
+            return self._api_call(method, endpoint)
 
         if method == 'post':
             payload['page'] = page