diff --git a/README.md b/README.md index 7d9f2af6..b4b398fc 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,38 @@ $ git clone https://github.com/frameio/python-frameio-client $ pip install . ``` -_Note: The Frame.io Python client may not work correctly in Python 3.8+_ +### Developing +Install the package into your development environment and link to it by running the following: + +```sh +pipenv install -e . -pre +``` ## Documentation [Frame.io API Documentation](https://developer.frame.io/docs) +### Use CLI +When you install this package, a cli tool called `fioctl` will also be installed to your environment. + +**To upload a file or folder** +```sh +fioctl \ +--token fio-u-YOUR_TOKEN_HERE \ +--destination "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \ +--target "YOUR LOCAL SYSTEM DIRECTORY" \ +--threads 8 +``` + +**To download a file, project, or folder** +```sh +fioctl \ +--token fio-u-YOUR_TOKEN_HERE \ +--destination "YOUR LOCAL SYSTEM DIRECTORY" \ +--target "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \ +--threads 2 +``` + ## Usage _Note: A valid token is required to make requests to Frame.io. Go to our [Developer Portal](https://developer.frame.io/) to get a token!_ diff --git a/examples/recursive_upload.py b/examples/recursive_upload.py new file mode 100644 index 00000000..05101eea --- /dev/null +++ b/examples/recursive_upload.py @@ -0,0 +1,94 @@ +import os +import time +import mimetypes +import concurrent.futures +import threading +from frameioclient import FrameioClient +from pprint import pprint + +global file_num +file_num = 0 + +global file_count +file_count = 0 + +def create_n_upload(task): + client=task[0] + file_p=task[1] + parent_asset_id=task[2] + abs_path = os.path.abspath(file_p) + file_s = os.path.getsize(file_p) + file_n = os.path.split(file_p)[1] + file_mime = mimetypes.guess_type(abs_path)[0] + + asset = client.create_asset( + parent_asset_id=parent_asset_id, + name=file_n, + type="file", + filetype=file_mime, + filesize=file_s + ) + + with open(abs_path, "rb") as ul_file: + asset_info = client.upload(asset, ul_file) + + return asset_info + + +def create_folder(folder_n, parent_asset_id): + asset = client.create_asset( + parent_asset_id=parent_asset_id, + name=folder_n, + type="folder", + ) + + return asset['id'] + + +def file_counter(root_folder): + matches = [] + for root, dirnames, filenames in os.walk(root_folder): + for filename in filenames: + matches.append(os.path.join(filename)) + + return matches + + +def recursive_upload(client, folder, parent_asset_id): + # Seperate files and folders: + file_list = list() + folder_list = list() + + for item in os.listdir(folder): + if item == ".DS_Store": # Ignore .DS_Store files on Mac + continue + + complete_item_path = os.path.join(folder, item) + + if os.path.isfile(complete_item_path): + file_list.append(item) + else: + folder_list.append(item) + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + for file_p in file_list: + global file_num + file_num += 1 + print(f"Starting {file_num}/{file_count}") + complete_dir_obj = os.path.join(folder, file_p) + task = (client, complete_dir_obj, parent_asset_id) + executor.submit(create_n_upload, task) + + for folder_i in folder_list: + new_folder = os.path.join(folder, folder_i) + new_parent_asset_id = create_folder(folder_i, parent_asset_id) + recursive_upload(client, new_folder, new_parent_asset_id) + + +if __name__ == "__main__": + root_folder = "./test_structure" + parent_asset_id = "PARENT_ASSET_ID" + client = FrameioClient(os.getenv("FRAME_IO_TOKEN")) + + file_count = len(file_counter(root_folder)) + recursive_upload(client, root_folder, parent_asset_id) \ No newline at end of file diff --git a/frameioclient/client.py b/frameioclient/client.py index 19f51a06..dd7e1976 100644 --- a/frameioclient/client.py +++ b/frameioclient/client.py @@ -8,8 +8,8 @@ ) class FrameioClient(APIClient, object): - def __init__(self, token, host='https://api.frame.io'): - super().__init__(token, host) + def __init__(self, token, host='https://api.frame.io', threads=5, progress=False): + super().__init__(token, host, threads, progress) @property def me(self): diff --git a/frameioclient/fiocli.py b/frameioclient/fiocli.py new file mode 100644 index 00000000..6d2ff98d --- /dev/null +++ b/frameioclient/fiocli.py @@ -0,0 +1,52 @@ +import os +import sys +import argparse + +from frameioclient import FrameioClient + + +def main(): + parser=argparse.ArgumentParser(prog='fiocli', description='Frame.io Python SDK CLI') + + ## Define args + parser.add_argument('--token', action='store', metavar='token', type=str, nargs='+', help='Developer Token') + # parser.add_argument('--op', action='store', metavar='op', type=str, nargs='+', help='Operation: upload, download') + parser.add_argument('--target', action='store', metavar='target', type=str, nargs='+', help='Target: remote project or folder, or alternatively a local file/folder') + parser.add_argument('--destination', action='store', metavar='destination', type=str, nargs='+', help='Destination: remote project or folder, or alternatively a local file/folder') + parser.add_argument('--threads', action='store', metavar='threads', type=int, nargs='+', help='Number of threads to use') + + ## Parse args + args = parser.parse_args() + + if args.threads: + threads = args.threads[0] + else: + threads = 5 + + ## Handle args + if args.token: + client = None + # print(args.token) + try: + client = FrameioClient(args.token[0], progress=True, threads=threads) + except Exception as e: + print("Failed") + sys.exit(1) + + # If args.op == 'upload': + if args.target: + if args.destination: + # Check to see if this is a local target and thus a download + if os.path.isdir(args.destination[0]): + asset = client.assets.get(args.target[0]) + return client.assets.download(asset, args.destination[0], progress=True, multi_part=True, concurrency=threads) + else: # This is an upload + if os.path.isdir(args.target[0]): + return client.assets.upload_folder(args.target[0], args.destination[0]) + else: + return client.assets.upload(args.destination[0], args.target[0]) + else: + print("No destination supplied") + else: + print("No target supplied") + diff --git a/frameioclient/lib/download.py b/frameioclient/lib/download.py index b593d3b5..0f57c926 100644 --- a/frameioclient/lib/download.py +++ b/frameioclient/lib/download.py @@ -35,6 +35,7 @@ def __init__(self, asset, download_folder, prefix, multi_part=False, replace=Fal self.futures = list() self.checksum = None self.original_checksum = None + self.checksum_verification = True self.chunk_size = (25 * 1024 * 1024) # 25 MB chunk size self.chunks = math.ceil(self.file_size/self.chunk_size) self.prefix = prefix @@ -45,6 +46,7 @@ def __init__(self, asset, download_folder, prefix, multi_part=False, replace=Fal self.session = self.aws_client._get_session(auth=None) self.filename = Utils.normalize_filename(asset["name"]) self.request_logs = list() + self.stats = True self._evaluate_asset() self._get_path() @@ -142,32 +144,34 @@ def download_handler(self): print("Destination folder not found, creating") os.mkdir(self.download_folder) - if not self.replace: - if os.path.isfile(self.get_path()): - print("File already exists at this location.") - return self.destination - else: - url = self.get_download_key() + if os.path.isfile(self.get_path()) == False: + pass - if self.watermarked == True: - return self.single_part_download(url) + if os.path.isfile(self.get_path()) and self.replace == True: + os.remove(self.get_path()) + + if os.path.isfile(self.get_path()) and self.replace == False: + print("File already exists at this location.") + return self.destination + + url = self.get_download_key() + + if self.watermarked == True: + return self.single_part_download(url) + else: + # Don't use multi-part download for files below 25 MB + if self.asset['filesize'] < 26214400: + return self.download(url) + if self.multi_part == True: + return self.multi_part_download(url) else: - # Don't use multi-part download for files below 25 MB - if self.asset['filesize'] < 26214400: - return self.download(url) - if self.multi_part == True: - return self.multi_part_download(url) - else: - return self.single_part_download(url) + return self.single_part_download(url) def single_part_download(self, url): start_time = time.time() print("Beginning download -- {} -- {}".format(self.asset["name"], Utils.format_bytes(self.file_size, type="size"))) # Downloading - r = self.session.get(url) - open(self.destination, "wb").write(r.content) - with open(self.destination, 'wb') as handle: try: # TODO make sure this approach works for SBWM download @@ -295,7 +299,7 @@ def multi_part_download(self, url): # Submit telemetry transfer_stats = {'speed': download_speed, 'time': download_time, 'cdn': AWSClient.check_cdn(url)} - Event(self.user_id, 'python-sdk-download-stats', transfer_stats) + # Event(self.user_id, 'python-sdk-download-stats', transfer_stats) # If stats = True, we return a dict with way more info, otherwise \ if self.stats: @@ -305,7 +309,7 @@ def multi_part_download(self, url): "speed": download_speed, "elapsed": download_time, "cdn": AWSClient.check_cdn(url), - "concurrency": self.concurrency, + "concurrency": self.aws_client.concurrency, "size": self.file_size, "chunks": self.chunks } diff --git a/frameioclient/lib/transport.py b/frameioclient/lib/transport.py index e7597d82..ed398561 100644 --- a/frameioclient/lib/transport.py +++ b/frameioclient/lib/transport.py @@ -46,24 +46,27 @@ def _get_session(self, auth=True): class APIClient(HTTPClient, object): - def __init__(self, token, host): + def __init__(self, token, host, threads, progress): super().__init__() self.host = host self.token = token + self.threads = threads + self.progress = progress self._initialize_thread() - self.session = self._get_session(auth=token) + self.session = self._get_session() self.auth_header = { - 'Authorization': 'Bearer {}'.format(self.token), + 'Authorization': 'Bearer {}'.format(self.token) } - def _api_call(self, method, endpoint, payload={}, limit=None): - url = '{}/v2{}'.format(self.host, endpoint) + def _format_api_call(self, endpoint): + return '{}/v2{}'.format(self.host, endpoint) + def _api_call(self, method, endpoint, payload={}, limit=None): headers = {**self.shared_headers, **self.auth_header} r = self.session.request( method, - url, + self._format_api_call(endpoint), headers=headers, json=payload ) @@ -98,14 +101,14 @@ def get_specific_page(self, method, endpoint, payload, page): Gets a specific page for that endpoint, used by Pagination Class :Args: - method (string): 'get', 'post' - endpoint (string): endpoint ('/accounts//teams') - payload (dict): Request payload - page (int): What page to get + method (string): 'get', 'post' + endpoint (string): endpoint ('/accounts//teams') + payload (dict): Request payload + page (int): What page to get """ if method == 'get': endpoint = '{}?page={}'.format(endpoint, page) - return self._api_call(method, endpoint) + return self._api_call(method, endpoint) if method == 'post': payload['page'] = page diff --git a/frameioclient/lib/upload.py b/frameioclient/lib/upload.py index 448dae1b..97d7f1ec 100644 --- a/frameioclient/lib/upload.py +++ b/frameioclient/lib/upload.py @@ -4,13 +4,17 @@ import threading import concurrent.futures +from .utils import Utils + thread_local = threading.local() class FrameioUploader(object): - def __init__(self, asset, file): + def __init__(self, asset=None, file=None): self.asset = asset self.file = file self.chunk_size = None + self.file_count = 0 + self.file_num = 0 def _calculate_chunks(self, total_size, chunk_count): self.chunk_size = int(math.ceil(total_size / chunk_count)) @@ -76,3 +80,50 @@ def upload(self): task = (url, chunk_offset, i) executor.submit(self._upload_chunk, task) + + + def file_counter(self, folder): + matches = [] + for root, dirnames, filenames in os.walk(folder): + for filename in filenames: + matches.append(os.path.join(filename)) + + self.file_count = len(matches) + + return matches + + def recursive_upload(self, client, folder, parent_asset_id): + # Seperate files and folders: + file_list = list() + folder_list = list() + + if self.file_count == 0: + self.file_counter(folder) + + for item in os.listdir(folder): + if item == ".DS_Store": # Ignore .DS_Store files on Mac + continue + + complete_item_path = os.path.join(folder, item) + + if os.path.isfile(complete_item_path): + file_list.append(item) + else: + folder_list.append(item) + + for file_p in file_list: + self.file_num += 1 + + complete_dir_obj = os.path.join(folder, file_p) + print(f"Starting {self.file_num:02d}/{self.file_count}, Size: {Utils.format_bytes(os.path.getsize(complete_dir_obj), type='size')}, Name: {file_p}") + client.assets.upload(parent_asset_id, complete_dir_obj) + + for folder_name in folder_list: + new_folder = os.path.join(folder, folder_name) + new_parent_asset_id = client.assets.create( + parent_asset_id=parent_asset_id, + name=folder_name, + type="folder" + )['id'] + + self.recursive_upload(client, new_folder, new_parent_asset_id) \ No newline at end of file diff --git a/frameioclient/services/assets.py b/frameioclient/services/assets.py index 445a7c80..064406a0 100644 --- a/frameioclient/services/assets.py +++ b/frameioclient/services/assets.py @@ -7,6 +7,18 @@ from ..lib import FrameioUploader, FrameioDownloader, constants class Asset(Service): + def _build_asset_info(self, filepath): + full_path = os.path.abspath(filepath) + + file_info = { + "filepath": full_path, + "filename": os.path.basename(full_path), + "filesize": os.path.getsize(full_path), + "mimetype": mimetypes.guess_type(full_path)[0] + } + + return file_info + def get(self, asset_id): """ Get an asset by id. @@ -124,7 +136,6 @@ def from_url(self, parent_asset_id, name, url): url="https://" ) """ - payload = { "name": name, "type": "file", @@ -179,9 +190,7 @@ def bulk_copy(self, destination_folder_id, asset_list=[], copy_comments=False): client.assets.bulk_copy("adeffee123342", asset_list=["7ee008c5-49a2-f8b5-997d-8b64de153c30", \ "7ee008c5-49a2-f8b5-997d-8b64de153c30"], copy_comments=True) """ - payload = {"batch": []} - new_list = list() if copy_comments: payload['copy_comments'] = "all" @@ -213,31 +222,9 @@ def _upload(self, asset, file): Example:: client.upload(asset, open('example.mp4')) """ - uploader = FrameioUploader(asset, file) uploader.upload() - # def upload_folder(self, destination_id, folderpath): - # try: - # if os.path.isdir(folderpath): - # # Good it's a directory, we can keep going - - # except OSError: - # if not os.path.exists(folderpath): - # sys.exit("Folder doesn't exist, exiting...") - - def build_asset_info(self, filepath): - full_path = os.path.abspath(filepath) - - file_info = { - "filepath": full_path, - "filename": os.path.basename(full_path), - "filesize": os.path.getsize(full_path), - "mimetype": mimetypes.guess_type(full_path)[0] - } - - return file_info - def upload(self, destination_id, filepath, asset=None): """ Upload a file. The method will exit once the file is downloaded. @@ -248,7 +235,6 @@ def upload(self, destination_id, filepath, asset=None): that you want to upload. Example:: - client.assets.upload('1231-12414-afasfaf-aklsajflaksjfla', "./file.mov") """ diff --git a/setup.py b/setup.py index f456b2ff..a378aded 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,11 @@ def run(self): 'bump2version', ] }, + entry_points ={ + 'console_scripts': [ + 'fiocli = frameioclient.fiocli:main' + ] + }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/tests/integration.py b/tests/integration.py index 7dd4bd07..ecd9e8ec 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -31,11 +31,11 @@ def init_client(): sys.exit(1) if environment == "PRODUCTION": - client = FrameioClient(token) + client = FrameioClient(token, threads=10) print("Client connection initialized.") else: - client = FrameioClient(token, host='https://api.dev.frame.io') + client = FrameioClient(token, host='https://api.dev.frame.io', threads=10) print("Client connection initialized.") return client @@ -107,7 +107,7 @@ def test_download(client, override=False): start_time = time.time() print("{}/{} Beginning to download: {}".format(count, len(asset_list), asset['name'])) - client.assets.download(asset, download_dir, multi_part=True, concurrency=10) + client.assets.download(asset, download_dir, multi_part=True) download_time = time.time() - start_time download_speed = Utils.format_bytes(ceil(asset['filesize']/(download_time)))