7

I am trying to make a html uploader for very large files using HTML5 and Web Workers. Atm it uploads slowly but consumes alot of memory. I think it is transferring the whole file to the memory when it adds it to the form. Heres the code: jswebworker.js:

 /*importScripts('webworkerFormData.js');*/

(function() {
// Export variable to the global scope
(this == undefined ? self : this)['FormData'] = FormData;

var ___send$rw = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype['send'] = function(data) {
    if (data instanceof FormData) {
        if (!data.__endedMultipart) data.__append('--' + data.boundary + '--\r\n');
        data.__endedMultipart = true;
        this.setRequestHeader('Content-Type', 'multipart/form-data; boundary=' + data.boundary);
        data = new Uint8Array(data.data).buffer;
    }
    // Invoke original XHR.send
    return ___send$rw.call(this, data);
};

function FormData() {
    // Force a Constructor
    if (!(this instanceof FormData)) return new FormData();
    // Generate a random boundary - This must be unique with respect to the form's contents.
    this.boundary = '------RWWorkerFormDataBoundary' + Math.random().toString(36);
    var internal_data = this.data = [];
    /**
    * Internal method.
    * @param inp String | ArrayBuffer | Uint8Array  Input
    */
    this.__append = function(inp) {
        var i=0, len;
        if (typeof inp === 'string') {
            for (len=inp.length; i<len; i++)
                internal_data.push(inp.charCodeAt(i) & 0xff);
        } else if (inp && inp.byteLength) {/*If ArrayBuffer or typed array */
            if (!('byteOffset' in inp))   /* If ArrayBuffer, wrap in view */ 
                inp = new Uint8Array(inp);
            for (len=inp.byteLength; i<len; i++)
                internal_data.push(inp[i] & 0xff);
        }
    };
}
/**
* @param name     String                                  Key name
* @param value    String|Blob|File|Uint8Array|ArrayBuffer Value
* @param filename String                                  Optional File name (when value is not a string).
**/
FormData.prototype['append'] = function(name, value, filename) {
    if (this.__endedMultipart) {
        // Truncate the closing boundary
        this.data.length -= this.boundary.length + 6;
        this.__endedMultipart = false;
    }
    var valueType = Object.prototype.toString.call(value),
        part = '--' + this.boundary + '\r\n' + 
            'Content-Disposition: form-data; name="' + name + '"';

    if (/^\[object (?:Blob|File)(?:Constructor)?\]$/.test(valueType)) {

        return this.append(name,
                        new Uint8Array(new FileReaderSync().readAsArrayBuffer(value)),
                        filename || value.name);
    } else if (/^\[object (?:Uint8Array|ArrayBuffer)(?:Constructor)?\]$/.test(valueType)) {
        part += '; filename="'+ (filename || 'blob').replace(/"/g,'%22') +'"\r\n';
        part += 'Content-Type: application/octet-stream\r\n\r\n';
        this.__append(part);
        this.__append(value);
        part = '\r\n';
    } else {
        part += '\r\n\r\n' + value + '\r\n';
    }
    this.__append(part);
};
})();

movies = [];
var timeStarted = 0;
uploadingVar = false;
const BYTES_PER_CHUNK = 64 * 1024 * 1024; 

function toTitleCase(str)
{
return str.replace(/\w\S*/g, function(txt){return txt.charAt(0).toUpperCase() + txt.substr(1);});
}

function newUpload(blobOrFile, moviename, filename, i, fileType, sizeFile) {
var xhr = new XMLHttpRequest();
path = '/moviehtml/newmupload.php?        moviename='+escape(moviename)+'&filename='+escape(filename)+'&num='+escape(i);
xhr.open('POST', path, false);
self.postMessage(blobOrFile.size);
var fd = new FormData();
//xhr.setRequestHeader('Content-Type', fileType)
//blobOrFile2 = FileReaderSync.readAsArrayBuffer(blobOrFile);
fd.append("files1", blobOrFile);
//fd.append("moviename", moviename);
//fd.append("filename", filename);
//fd.append("num",i);

seconds = new Date() / 1000;
xhr.send(fd);
self.postMessage(xhr.responseText)
self.postMessage({'type':'partial','done':i*BYTES_PER_CHUNK, 'started':timeStarted, 'total':sizeFile});
var finish = Date()/1000;
if (finish >= (seconds+100)){
    return false;
}
return true;

}
function newFileUpload(file, movieName, fileType, filename, exten){
if (movieName == movieName.match(/[a-zA-Z0-9\. \:]+/)){
    timeStarted = new Date().getTime();

var blob = file;// this.files[0];
//var filename = blob.name;
var moviename = toTitleCase(movieName);
 // 1MB chunk sizes.
const SIZE = blob.size;
//alert(SIZE + ' '+document.getElementById('fileToUpload').files[0].size)
var start = 0;
var end = BYTES_PER_CHUNK;
//alert(SIZE/BYTES_PER_CHUNK)
var i = 1;
while(start < SIZE) {
    wow = newUpload(blob.slice(start, end), moviename, filename, i, fileType, SIZE);

    start = end;
    end = start + BYTES_PER_CHUNK;
    i++;
}
var xhr = new XMLHttpRequest();
var fd2 = new FormData();
typeFile = filename.split('.').pop()
fd2.append("type", blob.type);
fd2.append("exten", typeFile);
fd2.append("moviename", moviename);
xhr.open('POST', '/moviehtml/finishedupload.php', false);
xhr.send(fd2);
}
}
function process(){
uploadingVar = true;
while(movies.length > 0) {

    upMovie = movies.pop();
    var file = upMovie[0];
    var movieName = upMovie[1];
    var fileType = upMovie[2];
    var filename = upMovie[3];
    var exten = upMovie[4];
    self.postMessage({'type':'start','size':file.size, 'name':movieName});

    newFileUpload(file, movieName, fileType, filename, exten);
    self.postMessage({'type':'finish','name':movieName})
    self.postMessage(movieName + " Uploaded Succesfully");
 }
 uploadingVar = false;
}

self.addEventListener('message', function(e) {
movies.push(e.data);
if (!uploadingVar){
process();

}
}, false);

My function that calls it:

var worker = new Worker('jswebworker.js');
function testUpload(){
//newFileUpload();
var file = document.getElementById('fileToUpload').files[0];
worker.postMessage([file,toTitleCase(document.getElementById('movieName').value),     file.type, file.name, file.name.split('.').pop()]);

}

This is the web page for a media server for my flat. I am hoping that there is a way to create a blob without loading all of the original into memory. Thanks for any help, Nick

2
  • Where you able to figure out a solution? Commented Jan 23, 2014 at 0:46
  • Why don't you provide simpler code? I believe more people would actually read it if you do so. Commented Mar 20, 2017 at 2:43

1 Answer 1

2

I think that this library (here is the github page for resumable.js as well) may already be doing what you are trying too accomplish. Since I don't know how big the files are that you are tying to upload I can't provide any insight as far as benchmarks go. I hope this helps you out.

Sign up to request clarification or add additional context in comments.

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.