IBM Aspera Transfer SDK を使用して、IBM Cloud Object Storage にモデルをアップロードするには、このスクリプトを使用します:
import grpc
import sys
import time, subprocess
import json
import os.path
import os.environ
import transfer_pb2 as transfer_manager
import transfer_pb2_grpc as transfer_manager_grpc
path_to_aspera = os.environ['path-to-aspera']
sys.path.append(f'{path_to_aspera}/connectors/python')
defstart_aspera_daemon():
config_path = f'/{path_to_aspera}/config/asperatransferd.json'
daemon_path = f'/{path_to_aspera}/bin/asperatransferd'# Start the daemon
process = subprocess.Popen([daemon_path, '--config', config_path])
print(f"Started Aspera Transfer SDK daemon with PID {process.pid}")
time.sleep(5) # Increased wait time for the daemon to start properlyreturn process
defrun():
try:
# create a connection to the transfer manager daemon
client = transfer_manager_grpc.TransferServiceStub(grpc.insecure_channel('localhost:55002'))
# create transfer spec
transfer_spec = {
"session_initiation":{
"icos":{
"bucket": os.environ['bucket-name'],
"api_key": os.environ['cos-api-key'],
"ibm_service_instance_id": os.environ['cos-service-instance-id'],
"ibm_service_endpoint": os.environ['cos-service-endpoint'] # example: https://s3.us-south.cloud-object-storage.appdomain.cloud
}
},
"direction": "send",
"title": "COS Upload",
"assets": {
"destination_root": "/model",
"paths": [
{
"source": os.environ['path-to-model-folder']
}
]
}
}
transfer_spec = json.dumps(transfer_spec)
# create a transfer request
transfer_request = transfer_manager.TransferRequest(
transferType=transfer_manager.FILE_REGULAR,
config=transfer_manager.TransferConfig(),
transferSpec=transfer_spec
)
# send start transfer request to transfer manager daemontry:
transfer_response = client.StartTransfer(transfer_request)
transfer_id = transfer_response.transferId
print(f"Transfer started with id {transfer_id}")
except grpc.RpcError as e:
print(f"Error starting transfer: {e.code()}: {e.details()}")
return# monitor transfer statustry:
# Monitor transfer statuswhileTrue:
response = client.MonitorTransfers(
transfer_manager.RegistrationRequest(
filters=[transfer_manager.RegistrationFilter(
transferId=[transfer_id]
)]))
for transfer_info in response:
print(f"Transfer info: {transfer_info}")
# Check transfer status in response
status = transfer_info.status
if status == transfer_manager.FAILED or status == transfer_manager.COMPLETED:
print(f"Transfer finished with status {status}")
return# Wait before polling again
time.sleep(5)
except grpc.RpcError as e:
print(f"Error monitoring transfer: {e.code()}: {e.details()}")
except Exception as e:
print(f"Unexpected error: {str(e)}", file=sys.stderr)
if __name__ == '__main__':
# Start the Aspera Transfer SDK daemon
daemon_process = start_aspera_daemon()
# Run the file transfer
run()
# Optionally, stop the Aspera daemon after transfer
daemon_process.terminate()
print("Aspera Transfer SDK daemon stopped")
以下の例の手順に従って、IBM Cloud Object Storage に Amazon Web Services コマンドライン・インターフェースを使用してモデルをアップロードしてください:
Amazon Web Services コマンドラインインターフェースをpipでインストールします:
pip install awscli
Copy to clipboardクリップボードにコピーされました
必要な環境変数を設定する:
export AWS_ACCESS_KEY_ID="<your access key>"export AWS_SECRET_ACCESS_KEY="<your secret access key>"export ENDPOINT="<s3 endpoint URL>"export BUCKET_NAME="<name of the bucket to upload the model>"
MODEL_FOLDER=${MODEL_NAME//\//-}# The name of the created folder is based on model name.export MODEL_FOLDER=${MODEL_FOLDER//./-}# Just in case, we're removing slashes and dots from it.
Copy to clipboardクリップボードにコピーされました
IBM Cloud Object Storage バケットにモデルを追加するには、Amazon Web Services コマンドラインインターフェイスを使用します: