Are you planning to move your existing GB’s or TB’s data to s3?
Below nodejs script might help you to get started and upload your data to S3.
PreRequsite:
Ensure that aws-sdk node module is configured. If not follow AWS documentation https://aws.amazon.com/sdk-for-node-js/
1 |
npm install s3 --save |
s3 Options:
Usually accessKeyId and secretAccessKey is enough to make it happening. But if you are behind proxy, use those commented s3Options fields accordingly.
Source Directory
1 |
localDir: "<localdir>", |
For windows directory delimiter needs tobe delimited, eg c:\\mysite\\uploads
For Linux you can use something like ‘/var/www/site/uploads’
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
var s3 = require('s3'); //s3 client var client = s3.createClient({ maxAsyncS3: 20, // this is the default s3RetryCount: 3, // this is the default s3RetryDelay: 1000, // this is the default multipartUploadThreshold: 20971520, // this is the default (20 MB) multipartUploadSize: 15728640, // this is the default (15 MB) s3Options: { 'accessKeyId': "<accesskey>", 'secretAccessKey': "<screct access key>", //'sessionToken': "<use only forPrivateVPCaccountsonly>", //'httpOptions': { 'proxy': 'http://proxy' }, //'sslEnabled': false, 'region': 'us-east-1' //endpoint : "some.endpoint.com.s3-website-ap-southeast-1.amazonaws.com", // s3BucketEndpoint:true // any other options are passed to new AWS.S3() // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property }, }); var assets = { localDir: "<localdir>",//windows: eg d:\\uploads, unix: /var/www/site/uploads deleteRemoved: true, // default false, whether to remove s3 objects // that have no corresponding local file. s3Params: { Bucket: "yourBuckerName", Prefix: "folderA/", // other options supported by putObject, except Body and ContentLength. // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property }, }; var assets_uploader = client.uploadDir(assets); assets_uploader.on('error', function(err) { console.error("unable to sync assets folder , error :", err.stack); }); assets_uploader.on('progress', function(err) { console.log("Processing the data sync for the direcotry assets ." , assets_uploader.progressMd5Amount,assets_uploader.progressAmount,assets_uploader.progressTotal) }); assets_uploader.on('end', function(err) { console.log("Done uploading the direcotry. assets !!!") }); |