diff --git a/cps/static/js/archive.js b/cps/static/js/archive/archive.js similarity index 77% rename from cps/static/js/archive.js rename to cps/static/js/archive/archive.js index 3a0ceef3..0694ea22 100644 --- a/cps/static/js/archive.js +++ b/cps/static/js/archive/archive.js @@ -17,7 +17,7 @@ bitjs.archive = bitjs.archive || {}; // =========================================================================== // Stolen from Closure because it's the best way to do Java-like inheritance. - bitjs.base = function(me, optMethodName, varArgs) { + bitjs.base = function(me, opt_methodName, var_args) { var caller = arguments.callee.caller; if (caller.superClass_) { // This is a constructor. Call the superclass constructor. @@ -27,12 +27,11 @@ bitjs.archive = bitjs.archive || {}; var args = Array.prototype.slice.call(arguments, 2); var foundCaller = false; - for (var ctor = me.constructor; - ctor; ctor = ctor.superClass_ && ctor.superClass_.constructor) { - if (ctor.prototype[optMethodName] === caller) { + for (var ctor = me.constructor; ctor; ctor = ctor.superClass_ && ctor.superClass_.constructor) { + if (ctor.prototype[opt_methodName] === caller) { foundCaller = true; } else if (foundCaller) { - return ctor.prototype[optMethodName].apply(me, args); + return ctor.prototype[opt_methodName].apply(me, args); } } @@ -40,8 +39,8 @@ bitjs.archive = bitjs.archive || {}; // then one of two things happened: // 1) The caller is an instance method. // 2) This method was not called by the right caller. - if (me[optMethodName] === caller) { - return me.constructor.prototype[optMethodName].apply(me, args); + if (me[opt_methodName] === caller) { + return me.constructor.prototype[opt_methodName].apply(me, args); } else { throw Error( "goog.base called from a method of one name " + @@ -50,10 +49,10 @@ bitjs.archive = bitjs.archive || {}; }; bitjs.inherits = function(childCtor, parentCtor) { /** @constructor */ - function TempCtor() {} - TempCtor.prototype = parentCtor.prototype; + function tempCtor() {}; + tempCtor.prototype = parentCtor.prototype; childCtor.superClass_ = parentCtor.prototype; - childCtor.prototype = new TempCtor(); + childCtor.prototype = new tempCtor(); childCtor.prototype.constructor = childCtor; }; // =========================================================================== @@ -66,10 +65,10 @@ bitjs.archive = bitjs.archive || {}; */ bitjs.archive.UnarchiveEvent = function(type) { /** - * The event type. - * - * @type {string} - */ + * The event type. + * + * @type {string} + */ this.type = type; }; @@ -94,10 +93,10 @@ bitjs.archive = bitjs.archive || {}; bitjs.base(this, bitjs.archive.UnarchiveEvent.Type.INFO); /** - * The information message. - * - * @type {string} - */ + * The information message. + * + * @type {string} + */ this.msg = msg; }; bitjs.inherits(bitjs.archive.UnarchiveInfoEvent, bitjs.archive.UnarchiveEvent); @@ -111,10 +110,10 @@ bitjs.archive = bitjs.archive || {}; bitjs.base(this, bitjs.archive.UnarchiveEvent.Type.ERROR); /** - * The information message. - * - * @type {string} - */ + * The information message. + * + * @type {string} + */ this.msg = msg; }; bitjs.inherits(bitjs.archive.UnarchiveErrorEvent, bitjs.archive.UnarchiveEvent); @@ -178,8 +177,8 @@ bitjs.archive = bitjs.archive || {}; bitjs.base(this, bitjs.archive.UnarchiveEvent.Type.EXTRACT); /** - * @type {UnarchivedFile} - */ + * @type {UnarchivedFile} + */ this.unarchivedFile = unarchivedFile; }; bitjs.inherits(bitjs.archive.UnarchiveExtractEvent, bitjs.archive.UnarchiveEvent); @@ -189,28 +188,28 @@ bitjs.archive = bitjs.archive || {}; * Base class for all Unarchivers. * * @param {ArrayBuffer} arrayBuffer The Array Buffer. - * @param {string} optPathToBitJS Optional string for where the BitJS files are located. + * @param {string} opt_pathToBitJS Optional string for where the BitJS files are located. * @constructor */ - bitjs.archive.Unarchiver = function(arrayBuffer, optPathToBitJS) { + bitjs.archive.Unarchiver = function(arrayBuffer, opt_pathToBitJS) { /** - * The ArrayBuffer object. - * @type {ArrayBuffer} - * @protected - */ + * The ArrayBuffer object. + * @type {ArrayBuffer} + * @protected + */ this.ab = arrayBuffer; /** - * The path to the BitJS files. - * @type {string} - * @private - */ - this.pathToBitJS_ = optPathToBitJS || ""; + * The path to the BitJS files. + * @type {string} + * @private + */ + this.pathToBitJS_ = opt_pathToBitJS || "/"; /** - * A map from event type to an array of listeners. - * @type {Map.} - */ + * A map from event type to an array of listeners. + * @type {Map.} + */ this.listeners_ = {}; for (var type in bitjs.archive.UnarchiveEvent.Type) { this.listeners_[bitjs.archive.UnarchiveEvent.Type[type]] = []; @@ -282,7 +281,7 @@ bitjs.archive = bitjs.archive || {}; /** * Starts the unarchive in a separate Web Worker thread and returns immediately. - */ + */ bitjs.archive.Unarchiver.prototype.start = function() { var me = this; var scriptFileName = this.pathToBitJS_ + this.getScriptFileName(); @@ -320,8 +319,8 @@ bitjs.archive = bitjs.archive || {}; * @extends {bitjs.archive.Unarchiver} * @constructor */ - bitjs.archive.Unzipper = function(arrayBuffer, optPathToBitJS) { - bitjs.base(this, arrayBuffer, optPathToBitJS); + bitjs.archive.Unzipper = function(arrayBuffer, opt_pathToBitJS) { + bitjs.base(this, arrayBuffer, opt_pathToBitJS); }; bitjs.inherits(bitjs.archive.Unzipper, bitjs.archive.Unarchiver); bitjs.archive.Unzipper.prototype.getScriptFileName = function() { @@ -333,8 +332,8 @@ bitjs.archive = bitjs.archive || {}; * @extends {bitjs.archive.Unarchiver} * @constructor */ - bitjs.archive.Unrarrer = function(arrayBuffer, optPathToBitJS) { - bitjs.base(this, arrayBuffer, optPathToBitJS); + bitjs.archive.Unrarrer = function(arrayBuffer, opt_pathToBitJS) { + bitjs.base(this, arrayBuffer, opt_pathToBitJS); }; bitjs.inherits(bitjs.archive.Unrarrer, bitjs.archive.Unarchiver); bitjs.archive.Unrarrer.prototype.getScriptFileName = function() { @@ -346,12 +345,35 @@ bitjs.archive = bitjs.archive || {}; * @extends {bitjs.archive.Unarchiver} * @constructor */ - bitjs.archive.Untarrer = function(arrayBuffer, optPathToBitJS) { - bitjs.base(this, arrayBuffer, optPathToBitJS); + bitjs.archive.Untarrer = function(arrayBuffer, opt_pathToBitJS) { + bitjs.base(this, arrayBuffer, opt_pathToBitJS); }; bitjs.inherits(bitjs.archive.Untarrer, bitjs.archive.Unarchiver); bitjs.archive.Untarrer.prototype.getScriptFileName = function() { return "untar.js"; }; + /** + * Factory method that creates an unarchiver based on the byte signature found + * in the arrayBuffer. + * @param {ArrayBuffer} ab + * @param {string=} opt_pathToBitJS Path to the unarchiver script files. + * @return {bitjs.archive.Unarchiver} + */ + bitjs.archive.GetUnarchiver = function(ab, opt_pathToBitJS) { + var unarchiver = null; + var pathToBitJS = opt_pathToBitJS || ''; + var h = new Uint8Array(ab, 0, 10); + + if (h[0] == 0x52 && h[1] == 0x61 && h[2] == 0x72 && h[3] == 0x21) { // Rar! + unarchiver = new bitjs.archive.Unrarrer(ab, pathToBitJS); + } else if (h[0] == 80 && h[1] == 75) { // PK (Zip) + unarchiver = new bitjs.archive.Unzipper(ab, pathToBitJS); + } else { // Try with tar + console.log('geter'); + unarchiver = new bitjs.archive.Untarrer(ab, pathToBitJS); + } + return unarchiver; + }; + })(); diff --git a/cps/static/js/archive/rarvm.js b/cps/static/js/archive/rarvm.js new file mode 100644 index 00000000..769c25be --- /dev/null +++ b/cps/static/js/archive/rarvm.js @@ -0,0 +1,858 @@ +/** + * rarvm.js + * + * Licensed under the MIT License + * + * Copyright(c) 2017 Google Inc. + */ + +/** + * CRC Implementation. + */ +var CRCTab = new Array(256).fill(0); + +function InitCRC() { + for (var i = 0; i < 256; ++i) { + var c = i; + for (var j = 0; j < 8; ++j) { + // Read http://stackoverflow.com/questions/6798111/bitwise-operations-on-32-bit-unsigned-ints + // for the bitwise operator issue (JS interprets operands as 32-bit signed + // integers and we need to deal with unsigned ones here). + c = ((c & 1) ? ((c >>> 1) ^ 0xEDB88320) : (c >>> 1)) >>> 0; + } + CRCTab[i] = c; + } +} + +/** + * @param {number} startCRC + * @param {Uint8Array} arr + * @return {number} + */ +function CRC(startCRC, arr) { + if (CRCTab[1] == 0) { + InitCRC(); + } + + /* + #if defined(LITTLE_ENDIAN) && defined(PRESENT_INT32) && defined(ALLOW_NOT_ALIGNED_INT) + while (Size>0 && ((long)Data & 7)) + { + StartCRC=CRCTab[(byte)(StartCRC^Data[0])]^(StartCRC>>8); + Size--; + Data++; + } + while (Size>=8) + { + StartCRC^=*(uint32 *)Data; + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + StartCRC^=*(uint32 *)(Data+4); + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + StartCRC=CRCTab[(byte)StartCRC]^(StartCRC>>8); + Data+=8; + Size-=8; + } + #endif + */ + + for (var i = 0; i < arr.length; ++i) { + var byte = ((startCRC ^ arr[i]) >>> 0) & 0xff; + startCRC = (CRCTab[byte] ^ (startCRC >>> 8)) >>> 0; + } + + return startCRC; +} + +// ============================================================================================== // + + +/** + * RarVM Implementation. + */ +var VM_MEMSIZE = 0x40000; +var VM_MEMMASK = (VM_MEMSIZE - 1); +var VM_GLOBALMEMADDR = 0x3C000; +var VM_GLOBALMEMSIZE = 0x2000; +var VM_FIXEDGLOBALSIZE = 64; +var MAXWINSIZE = 0x400000; +var MAXWINMASK = (MAXWINSIZE - 1); + +/** + */ +var VM_Commands = { + VM_MOV: 0, + VM_CMP: 1, + VM_ADD: 2, + VM_SUB: 3, + VM_JZ: 4, + VM_JNZ: 5, + VM_INC: 6, + VM_DEC: 7, + VM_JMP: 8, + VM_XOR: 9, + VM_AND: 10, + VM_OR: 11, + VM_TEST: 12, + VM_JS: 13, + VM_JNS: 14, + VM_JB: 15, + VM_JBE: 16, + VM_JA: 17, + VM_JAE: 18, + VM_PUSH: 19, + VM_POP: 20, + VM_CALL: 21, + VM_RET: 22, + VM_NOT: 23, + VM_SHL: 24, + VM_SHR: 25, + VM_SAR: 26, + VM_NEG: 27, + VM_PUSHA: 28, + VM_POPA: 29, + VM_PUSHF: 30, + VM_POPF: 31, + VM_MOVZX: 32, + VM_MOVSX: 33, + VM_XCHG: 34, + VM_MUL: 35, + VM_DIV: 36, + VM_ADC: 37, + VM_SBB: 38, + VM_PRINT: 39, + + /* + #ifdef VM_OPTIMIZE + VM_MOVB, VM_MOVD, VM_CMPB, VM_CMPD, + + VM_ADDB, VM_ADDD, VM_SUBB, VM_SUBD, VM_INCB, VM_INCD, VM_DECB, VM_DECD, + VM_NEGB, VM_NEGD, + #endif + */ + + // TODO: This enum value would be much larger if VM_OPTIMIZE. + VM_STANDARD: 40, +}; + +/** + */ +var VM_StandardFilters = { + VMSF_NONE: 0, + VMSF_E8: 1, + VMSF_E8E9: 2, + VMSF_ITANIUM: 3, + VMSF_RGB: 4, + VMSF_AUDIO: 5, + VMSF_DELTA: 6, + VMSF_UPCASE: 7, +}; + +/** + */ +var VM_Flags = { + VM_FC: 1, + VM_FZ: 2, + VM_FS: 0x80000000, +}; + +/** + */ +var VM_OpType = { + VM_OPREG: 0, + VM_OPINT: 1, + VM_OPREGMEM: 2, + VM_OPNONE: 3, +}; + +/** + * Finds the key that maps to a given value in an object. This function is useful in debugging + * variables that use the above enums. + * @param {Object} obj + * @param {number} val + * @return {string} The key/enum value as a string. + */ +function findKeyForValue(obj, val) { + for (var key in obj) { + if (obj[key] === val) { + return key; + } + } + return null; +} + +function getDebugString(obj, val) { + var s = 'Unknown.'; + if (obj === VM_Commands) { + s = 'VM_Commands.'; + } else if (obj === VM_StandardFilters) { + s = 'VM_StandardFilters.'; + } else if (obj === VM_Flags) { + s = 'VM_OpType.'; + } else if (obj === VM_OpType) { + s = 'VM_OpType.'; + } + + return s + findKeyForValue(obj, val); +} + +/** + * @struct + * @constructor + */ +var VM_PreparedOperand = function() { + /** @type {VM_OpType} */ + this.Type; + + /** @type {number} */ + this.Data = 0; + + /** @type {number} */ + this.Base = 0; + + // TODO: In C++ this is a uint* + /** @type {Array} */ + this.Addr = null; +}; + +/** @return {string} */ +VM_PreparedOperand.prototype.toString = function() { + if (this.Type === null) { + return 'Error: Type was null in VM_PreparedOperand'; + } + return '{ ' + + 'Type: ' + getDebugString(VM_OpType, this.Type) + + ', Data: ' + this.Data + + ', Base: ' + this.Base + + ' }'; +}; + +/** + * @struct + * @constructor + */ +var VM_PreparedCommand = function() { + /** @type {VM_Commands} */ + this.OpCode; + + /** @type {boolean} */ + this.ByteMode = false; + + /** @type {VM_PreparedOperand} */ + this.Op1 = new VM_PreparedOperand(); + + /** @type {VM_PreparedOperand} */ + this.Op2 = new VM_PreparedOperand(); +}; + +/** @return {string} */ +VM_PreparedCommand.prototype.toString = function(indent) { + if (this.OpCode === null) { + return 'Error: OpCode was null in VM_PreparedCommand'; + } + indent = indent || ''; + return indent + '{\n' + + indent + ' OpCode: ' + getDebugString(VM_Commands, this.OpCode) + ',\n' + + indent + ' ByteMode: ' + this.ByteMode + ',\n' + + indent + ' Op1: ' + this.Op1.toString() + ',\n' + + indent + ' Op2: ' + this.Op2.toString() + ',\n' + + indent + '}'; +}; + +/** + * @struct + * @constructor + */ +var VM_PreparedProgram = function() { + /** @type {Array} */ + this.Cmd = []; + + /** @type {Array} */ + this.AltCmd = null; + + /** @type {Uint8Array} */ + this.GlobalData = new Uint8Array(); + + /** @type {Uint8Array} */ + this.StaticData = new Uint8Array(); // static data contained in DB operators + + /** @type {Uint32Array} */ + this.InitR = new Uint32Array(7); + + /** + * A pointer to bytes that have been filtered by a program. + * @type {Uint8Array} + */ + this.FilteredData = null; +}; + +/** @return {string} */ +VM_PreparedProgram.prototype.toString = function() { + var s = '{\n Cmd: [\n'; + for (var i = 0; i < this.Cmd.length; ++i) { + s += this.Cmd[i].toString(' ') + ',\n'; + } + s += '],\n'; + // TODO: Dump GlobalData, StaticData, InitR? + s += ' }\n'; + return s; +}; + +/** + * @struct + * @constructor + */ +var UnpackFilter = function() { + /** @type {number} */ + this.BlockStart = 0; + + /** @type {number} */ + this.BlockLength = 0; + + /** @type {number} */ + this.ExecCount = 0; + + /** @type {boolean} */ + this.NextWindow = false; + + // position of parent filter in Filters array used as prototype for filter + // in PrgStack array. Not defined for filters in Filters array. + /** @type {number} */ + this.ParentFilter = null; + + /** @type {VM_PreparedProgram} */ + this.Prg = new VM_PreparedProgram(); +}; + +var VMCF_OP0 = 0; +var VMCF_OP1 = 1; +var VMCF_OP2 = 2; +var VMCF_OPMASK = 3; +var VMCF_BYTEMODE = 4; +var VMCF_JUMP = 8; +var VMCF_PROC = 16; +var VMCF_USEFLAGS = 32; +var VMCF_CHFLAGS = 64; + +var VM_CmdFlags = [ + /* VM_MOV */ + VMCF_OP2 | VMCF_BYTEMODE, + /* VM_CMP */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_ADD */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_SUB */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_JZ */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_JNZ */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_INC */ + VMCF_OP1 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_DEC */ + VMCF_OP1 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_JMP */ + VMCF_OP1 | VMCF_JUMP, + /* VM_XOR */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_AND */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_OR */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_TEST */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_JS */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_JNS */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_JB */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_JBE */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_JA */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_JAE */ + VMCF_OP1 | VMCF_JUMP | VMCF_USEFLAGS, + /* VM_PUSH */ + VMCF_OP1, + /* VM_POP */ + VMCF_OP1, + /* VM_CALL */ + VMCF_OP1 | VMCF_PROC, + /* VM_RET */ + VMCF_OP0 | VMCF_PROC, + /* VM_NOT */ + VMCF_OP1 | VMCF_BYTEMODE, + /* VM_SHL */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_SHR */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_SAR */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_NEG */ + VMCF_OP1 | VMCF_BYTEMODE | VMCF_CHFLAGS, + /* VM_PUSHA */ + VMCF_OP0, + /* VM_POPA */ + VMCF_OP0, + /* VM_PUSHF */ + VMCF_OP0 | VMCF_USEFLAGS, + /* VM_POPF */ + VMCF_OP0 | VMCF_CHFLAGS, + /* VM_MOVZX */ + VMCF_OP2, + /* VM_MOVSX */ + VMCF_OP2, + /* VM_XCHG */ + VMCF_OP2 | VMCF_BYTEMODE, + /* VM_MUL */ + VMCF_OP2 | VMCF_BYTEMODE, + /* VM_DIV */ + VMCF_OP2 | VMCF_BYTEMODE, + /* VM_ADC */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_USEFLAGS | VMCF_CHFLAGS, + /* VM_SBB */ + VMCF_OP2 | VMCF_BYTEMODE | VMCF_USEFLAGS | VMCF_CHFLAGS, + /* VM_PRINT */ + VMCF_OP0, +]; + + +/** + * @param {number} length + * @param {number} crc + * @param {VM_StandardFilters} type + * @struct + * @constructor + */ +var StandardFilterSignature = function(length, crc, type) { + /** @type {number} */ + this.Length = length; + + /** @type {number} */ + this.CRC = crc; + + /** @type {VM_StandardFilters} */ + this.Type = type; +}; + +/** + * @type {Array} + */ +var StdList = [ + new StandardFilterSignature(53, 0xad576887, VM_StandardFilters.VMSF_E8), + new StandardFilterSignature(57, 0x3cd7e57e, VM_StandardFilters.VMSF_E8E9), + new StandardFilterSignature(120, 0x3769893f, VM_StandardFilters.VMSF_ITANIUM), + new StandardFilterSignature(29, 0x0e06077d, VM_StandardFilters.VMSF_DELTA), + new StandardFilterSignature(149, 0x1c2c5dc8, VM_StandardFilters.VMSF_RGB), + new StandardFilterSignature(216, 0xbc85e701, VM_StandardFilters.VMSF_AUDIO), + new StandardFilterSignature(40, 0x46b9c560, VM_StandardFilters.VMSF_UPCASE), +]; + +/** + * @constructor + */ +var RarVM = function() { + /** @private {Uint8Array} */ + this.mem_ = null; + + /** @private {Uint32Array} */ + this.R_ = new Uint32Array(8); + + /** @private {number} */ + this.flags_ = 0; +}; + +/** + * Initializes the memory of the VM. + */ +RarVM.prototype.init = function() { + if (!this.mem_) { + this.mem_ = new Uint8Array(VM_MEMSIZE); + } +}; + +/** + * @param {Uint8Array} code + * @return {VM_StandardFilters} + */ +RarVM.prototype.isStandardFilter = function(code) { + var codeCRC = (CRC(0xffffffff, code, code.length) ^ 0xffffffff) >>> 0; + for (var i = 0; i < StdList.length; ++i) { + if (StdList[i].CRC == codeCRC && StdList[i].Length == code.length) + return StdList[i].Type; + } + + return VM_StandardFilters.VMSF_NONE; +}; + +/** + * @param {VM_PreparedOperand} op + * @param {boolean} byteMode + * @param {bitjs.io.BitStream} bstream A rtl bit stream. + */ +RarVM.prototype.decodeArg = function(op, byteMode, bstream) { + var data = bstream.peekBits(16); + if (data & 0x8000) { + op.Type = VM_OpType.VM_OPREG; // Operand is register (R[0]..R[7]) + bstream.readBits(1); // 1 flag bit and... + op.Data = bstream.readBits(3); // ... 3 register number bits + op.Addr = [this.R_[op.Data]] // TODO &R[Op.Data] // Register address + } else { + if ((data & 0xc000) == 0) { + op.Type = VM_OpType.VM_OPINT; // Operand is integer + bstream.readBits(2); // 2 flag bits + if (byteMode) { + op.Data = bstream.readBits(8); // Byte integer. + } else { + op.Data = RarVM.readData(bstream); // 32 bit integer. + } + } else { + // Operand is data addressed by register data, base address or both. + op.Type = VM_OpType.VM_OPREGMEM; + if ((data & 0x2000) == 0) { + bstream.readBits(3); // 3 flag bits + // Base address is zero, just use the address from register. + op.Data = bstream.readBits(3); // (Data>>10)&7 + op.Addr = [this.R_[op.Data]]; // TODO &R[op.Data] + op.Base = 0; + } else { + bstream.readBits(4); // 4 flag bits + if ((data & 0x1000) == 0) { + // Use both register and base address. + op.Data = bstream.readBits(3); + op.Addr = [this.R_[op.Data]]; // TODO &R[op.Data] + } else { + // Use base address only. Access memory by fixed address. + op.Data = 0; + } + op.Base = RarVM.readData(bstream); // Read base address. + } + } + } +}; + +/** + * @param {VM_PreparedProgram} prg + */ +RarVM.prototype.execute = function(prg) { + this.R_.set(prg.InitR); + + var globalSize = Math.min(prg.GlobalData.length, VM_GLOBALMEMSIZE); + if (globalSize) { + this.mem_.set(prg.GlobalData.subarray(0, globalSize), VM_GLOBALMEMADDR); + } + + var staticSize = Math.min(prg.StaticData.length, VM_GLOBALMEMSIZE - globalSize); + if (staticSize) { + this.mem_.set(prg.StaticData.subarray(0, staticSize), VM_GLOBALMEMADDR + globalSize); + } + + this.R_[7] = VM_MEMSIZE; + this.flags_ = 0; + + var preparedCodes = prg.AltCmd ? prg.AltCmd : prg.Cmd; + if (prg.Cmd.length > 0 && !this.executeCode(preparedCodes)) { + // Invalid VM program. Let's replace it with 'return' command. + preparedCode.OpCode = VM_Commands.VM_RET; + } + + var dataView = new DataView(this.mem_.buffer, VM_GLOBALMEMADDR); + var newBlockPos = dataView.getUint32(0x20, true /* little endian */ ) & VM_MEMMASK; + var newBlockSize = dataView.getUint32(0x1c, true /* little endian */ ) & VM_MEMMASK; + if (newBlockPos + newBlockSize >= VM_MEMSIZE) { + newBlockPos = newBlockSize = 0; + } + prg.FilteredData = this.mem_.subarray(newBlockPos, newBlockPos + newBlockSize); + + prg.GlobalData = new Uint8Array(0); + + var dataSize = Math.min(dataView.getUint32(0x30), + (VM_GLOBALMEMSIZE - VM_FIXEDGLOBALSIZE)); + if (dataSize != 0) { + var len = dataSize + VM_FIXEDGLOBALSIZE; + prg.GlobalData = new Uint8Array(len); + prg.GlobalData.set(mem.subarray(VM_GLOBALMEMADDR, VM_GLOBALMEMADDR + len)); + } +}; + +/** + * @param {Array} preparedCodes + * @return {boolean} + */ +RarVM.prototype.executeCode = function(preparedCodes) { + var codeIndex = 0; + var cmd = preparedCodes[codeIndex]; + // TODO: Why is this an infinite loop instead of just returning + // when a VM_RET is hit? + while (1) { + switch (cmd.OpCode) { + case VM_Commands.VM_RET: + if (this.R_[7] >= VM_MEMSIZE) { + return true; + } + //SET_IP(GET_VALUE(false,(uint *)&Mem[R[7] & VM_MEMMASK])); + this.R_[7] += 4; + continue; + + case VM_Commands.VM_STANDARD: + this.executeStandardFilter(cmd.Op1.Data); + break; + + default: + console.error('RarVM OpCode not supported: ' + getDebugString(VM_Commands, cmd.OpCode)); + break; + } // switch (cmd.OpCode) + codeIndex++; + cmd = preparedCodes[codeIndex]; + } +}; + +/** + * @param {number} filterType + */ +RarVM.prototype.executeStandardFilter = function(filterType) { + switch (filterType) { + case VM_StandardFilters.VMSF_DELTA: + var dataSize = this.R_[4]; + var channels = this.R_[0]; + var srcPos = 0; + var border = dataSize * 2; + + //SET_VALUE(false,&Mem[VM_GLOBALMEMADDR+0x20],DataSize); + var dataView = new DataView(this.mem_.buffer, VM_GLOBALMEMADDR); + dataView.setUint32(0x20, dataSize, true /* little endian */ ); + + if (dataSize >= VM_GLOBALMEMADDR / 2) { + break; + } + + // Bytes from same channels are grouped to continual data blocks, + // so we need to place them back to their interleaving positions. + for (var curChannel = 0; curChannel < channels; ++curChannel) { + var prevByte = 0; + for (var destPos = dataSize + curChannel; destPos < border; destPos += channels) { + prevByte = (prevByte - this.mem_[srcPos++]) & 0xff; + this.mem_[destPos] = prevByte; + } + } + + break; + + default: + console.error('RarVM Standard Filter not supported: ' + getDebugString(VM_StandardFilters, filterType)); + break; + } +}; + +/** + * @param {Uint8Array} code + * @param {VM_PreparedProgram} prg + */ +RarVM.prototype.prepare = function(code, prg) { + var codeSize = code.length; + + //InitBitInput(); + //memcpy(InBuf,Code,Min(CodeSize,BitInput::MAX_SIZE)); + var bstream = new bitjs.io.BitStream(code.buffer, true /* rtl */ ); + + // Calculate the single byte XOR checksum to check validity of VM code. + var xorSum = 0; + for (var i = 1; i < codeSize; ++i) { + xorSum ^= code[i]; + } + + bstream.readBits(8); + + prg.Cmd = []; // TODO: Is this right? I don't see it being done in rarvm.cpp. + + // VM code is valid if equal. + if (xorSum == code[0]) { + var filterType = this.isStandardFilter(code); + if (filterType != VM_StandardFilters.VMSF_NONE) { + // VM code is found among standard filters. + var curCmd = new VM_PreparedCommand(); + prg.Cmd.push(curCmd); + + curCmd.OpCode = VM_Commands.VM_STANDARD; + curCmd.Op1.Data = filterType; + // TODO: Addr=&CurCmd->Op1.Data + curCmd.Op1.Addr = [curCmd.Op1.Data]; + curCmd.Op2.Addr = [null]; // &CurCmd->Op2.Data; + curCmd.Op1.Type = VM_OpType.VM_OPNONE; + curCmd.Op2.Type = VM_OpType.VM_OPNONE; + codeSize = 0; + } + + var dataFlag = bstream.readBits(1); + + // Read static data contained in DB operators. This data cannot be + // changed, it is a part of VM code, not a filter parameter. + + if (dataFlag & 0x8000) { + var dataSize = RarVM.readData(bstream) + 1; + // TODO: This accesses the byte pointer of the bstream directly. Is that ok? + for (var i = 0; i < bstream.bytePtr < codeSize && i < dataSize; ++i) { + // Append a byte to the program's static data. + var newStaticData = new Uint8Array(prg.StaticData.length + 1); + newStaticData.set(prg.StaticData); + newStaticData[newStaticData.length - 1] = bstream.readBits(8); + prg.StaticData = newStaticData; + } + } + + while (bstream.bytePtr < codeSize) { + var curCmd = new VM_PreparedCommand(); + prg.Cmd.push(curCmd); // Prg->Cmd.Add(1) + var flag = bstream.peekBits(1); + if (!flag) { // (Data&0x8000)==0 + curCmd.OpCode = bstream.readBits(4); + } else { + curCmd.OpCode = (bstream.readBits(6) - 24); + } + + if (VM_CmdFlags[curCmd.OpCode] & VMCF_BYTEMODE) { + curCmd.ByteMode = (bstream.readBits(1) != 0); + } else { + curCmd.ByteMode = 0; + } + curCmd.Op1.Type = VM_OpType.VM_OPNONE; + curCmd.Op2.Type = VM_OpType.VM_OPNONE; + var opNum = (VM_CmdFlags[curCmd.OpCode] & VMCF_OPMASK); + curCmd.Op1.Addr = null; + curCmd.Op2.Addr = null; + if (opNum > 0) { + this.decodeArg(curCmd.Op1, curCmd.ByteMode, bstream); // reading the first operand + if (opNum == 2) { + this.decodeArg(curCmd.Op2, curCmd.ByteMode, bstream); // reading the second operand + } else { + if (curCmd.Op1.Type == VM_OpType.VM_OPINT && (VM_CmdFlags[curCmd.OpCode] & (VMCF_JUMP | VMCF_PROC))) { + // Calculating jump distance. + var distance = curCmd.Op1.Data; + if (distance >= 256) { + distance -= 256; + } else { + if (distance >= 136) { + distance -= 264; + } else { + if (distance >= 16) { + distance -= 8; + } else { + if (distance >= 8) { + distance -= 16; + } + } + } + distance += prg.Cmd.length; + } + curCmd.Op1.Data = distance; + } + } + } // if (OpNum>0) + } // while ((uint)InAddrOp1.Data + curCmd.Op1.Addr = [curCmd.Op1.Data]; + curCmd.Op2.Addr = [curCmd.Op2.Data]; + curCmd.Op1.Type = VM_OpType.VM_OPNONE; + curCmd.Op2.Type = VM_OpType.VM_OPNONE; + + // If operand 'Addr' field has not been set by DecodeArg calls above, + // let's set it to point to operand 'Data' field. It is necessary for + // VM_OPINT type operands (usual integers) or maybe if something was + // not set properly for other operands. 'Addr' field is required + // for quicker addressing of operand data. + for (var i = 0; i < prg.Cmd.length; ++i) { + var cmd = prg.Cmd[i]; + if (cmd.Op1.Addr == null) { + cmd.Op1.Addr = [cmd.Op1.Data]; + } + if (cmd.Op2.Addr == null) { + cmd.Op2.Addr = [cmd.Op2.Data]; + } + } + + /* + #ifdef VM_OPTIMIZE + if (CodeSize!=0) + Optimize(Prg); + #endif + */ +}; + +/** + * @param {Uint8Array} arr The byte array to set a value in. + * @param {number} value The unsigned 32-bit value to set. + * @param {number} offset Offset into arr to start setting the value, defaults to 0. + */ +RarVM.prototype.setLowEndianValue = function(arr, value, offset) { + var i = offset || 0; + arr[i] = value & 0xff; + arr[i + 1] = (value >>> 8) & 0xff; + arr[i + 2] = (value >>> 16) & 0xff; + arr[i + 3] = (value >>> 24) & 0xff; +}; + +/** + * Sets a number of bytes of the VM memory at the given position from a + * source buffer of bytes. + * @param {number} pos The position in the VM memory to start writing to. + * @param {Uint8Array} buffer The source buffer of bytes. + * @param {number} dataSize The number of bytes to set. + */ +RarVM.prototype.setMemory = function(pos, buffer, dataSize) { + if (pos < VM_MEMSIZE) { + var numBytes = Math.min(dataSize, VM_MEMSIZE - pos); + for (var i = 0; i < numBytes; ++i) { + this.mem_[pos + i] = buffer[i]; + } + } +}; + +/** + * Static function that reads in the next set of bits for the VM + * (might return 4, 8, 16 or 32 bits). + * @param {bitjs.io.BitStream} bstream A RTL bit stream. + * @return {number} The value of the bits read. + */ +RarVM.readData = function(bstream) { + // Read in the first 2 bits. + var flags = bstream.readBits(2); + switch (flags) { // Data&0xc000 + // Return the next 4 bits. + case 0: + return bstream.readBits(4); // (Data>>10)&0xf + + case 1: // 0x4000 + // 0x3c00 => 0011 1100 0000 0000 + if (bstream.peekBits(4) == 0) { // (Data&0x3c00)==0 + // Skip the 4 zero bits. + bstream.readBits(4); + // Read in the next 8 and pad with 1s to 32 bits. + return (0xffffff00 | bstream.readBits(8)) >>> 0; // ((Data>>2)&0xff) + } + + // Else, read in the next 8. + return bstream.readBits(8); + + // Read in the next 16. + case 2: // 0x8000 + var val = bstream.getBits(); + bstream.readBits(16); + return val; //bstream.readBits(16); + + // case 3 + default: + return (bstream.readBits(16) << 16) | bstream.readBits(16); + } +}; + +// ============================================================================================== // \ No newline at end of file diff --git a/cps/static/js/unrar.js b/cps/static/js/archive/unrar.js similarity index 56% rename from cps/static/js/unrar.js rename to cps/static/js/archive/unrar.js index 30263bb2..1094a42b 100644 --- a/cps/static/js/unrar.js +++ b/cps/static/js/archive/unrar.js @@ -1,6 +1,8 @@ /** * unrar.js * + * Licensed under the MIT License + * * Copyright(c) 2011 Google Inc. * Copyright(c) 2011 antimatter15 * @@ -11,8 +13,10 @@ /* global bitjs, importScripts */ // This file expects to be invoked as a Worker (see onmessage below). -importScripts("io.js"); -importScripts("archive.js"); +importScripts('../io/bitstream.js'); +importScripts('../io/bytebuffer.js'); +importScripts('archive.js'); +importScripts('rarvm.js'); // Progress variables. var currentFilename = ""; @@ -59,11 +63,15 @@ var MAIN_HEAD = 0x73, // PROTECT_HEAD = 0x78, // SIGN_HEAD = 0x79, // NEWSUB_HEAD = 0x7a, - ENDARC_HEAD = 0x7b; + ENDARC_HEAD = 0x7b; -// bstream is a bit stream -var RarVolumeHeader = function(bstream) { +// ============================================================================================== // +/** + * @param {bitjs.io.BitStream} bstream + * @constructor + */ +var RarVolumeHeader = function(bstream) { var headPos = bstream.bytePtr; // byte 1,2 info("Rar Volume Header @" + bstream.bytePtr); @@ -197,15 +205,18 @@ var RarVolumeHeader = function(bstream) { } - while (headPos + this.headSize > bstream.bytePtr) bstream.readBits(1); + while (headPos + this.headSize > bstream.bytePtr) { + bstream.readBits(1); + } - info("Found FILE_HEAD with packSize=" + this.packSize + ", unpackedSize= " + this.unpackedSize + ", hostOS=" + this.hostOS + ", unpVer=" + this.unpVer + ", method=" + this.method + ", filename=" + this.filename); + // If Info line is commented in firefox fails if server on same computer than browser with error "expected expression, got default" + //info("Found FILE_HEAD with packSize=" + this.packSize + ", unpackedSize= " + this.unpackedSize + ", hostOS=" + this.hostOS + ", unpVer=" + this.unpVer + ", method=" + this.method + ", filename=" + this.filename); break; default: info("Found a header of type 0x" + byteValueToHexString(this.headType)); // skip the rest of the header bytes (for now) - bstream.readBytes( this.headSize - 7 ); + bstream.readBytes(this.headSize - 7); break; } }; @@ -213,20 +224,22 @@ var RarVolumeHeader = function(bstream) { //var BLOCK_LZ = 0; var rLDecode = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224], - rLBits = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5], + rLBits = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5], rDBitLengthCounts = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 14, 0, 12], rSDDecode = [0, 4, 8, 16, 32, 64, 128, 192], - rSDBits = [2, 2, 3, 4, 5, 6, 6, 6]; + rSDBits = [2, 2, 3, 4, 5, 6, 6, 6]; var rDDecode = [0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, 262144, 327680, 393216, 458752, 524288, 589824, - 655360, 720896, 786432, 851968, 917504, 983040]; + 655360, 720896, 786432, 851968, 917504, 983040 +]; var rDBits = [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, - 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16]; + 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16 +]; var rLowDistRepCount = 16; @@ -266,15 +279,45 @@ var RD = { //rep decode DecodeNum: new Array(rRC) }; +/** + * @type {Array} + */ +var rOldBuffers = []; + +/** + * The current buffer we are unpacking to. + * @type {bitjs.io.ByteBuffer} + */ var rBuffer; -// read in Huffman tables for RAR +/** + * The buffer of the final bytes after filtering (only used in Unpack29). + * @type {bitjs.io.ByteBuffer} + */ +var wBuffer; + + +/** + * In unpack.cpp, UnpPtr keeps track of what bytes have been unpacked + * into the Window buffer and WrPtr keeps track of what bytes have been + * actually written to disk after the unpacking and optional filtering + * has been done. + * + * In our case, rBuffer is the buffer for the unpacked bytes and wBuffer is + * the final output bytes. + */ + + +/** + * Read in Huffman tables for RAR + * @param {bitjs.io.BitStream} bstream + */ function rarReadTables(bstream) { - var BitLength = new Array(rBC), - Table = new Array(rHuffTableSize); + var BitLength = new Array(rBC); + var Table = new Array(rHuffTableSize); var i; // before we start anything we need to get byte-aligned - bstream.readBits( (8 - bstream.bitPtr) & 0x7 ); + bstream.readBits((8 - bstream.bitPtr) & 0x7); if (bstream.readBits(1)) { info("Error! PPM not implemented yet"); @@ -282,12 +325,13 @@ function rarReadTables(bstream) { } if (!bstream.readBits(1)) { //discard old table - for (i = UnpOldTable.length; i--;) UnpOldTable[i] = 0; + for (i = UnpOldTable.length; i--;) { + UnpOldTable[i] = 0; + } } // read in bit lengths for (var I = 0; I < rBC; ++I) { - var Length = bstream.readBits(4); if (Length === 15) { var ZeroCount = bstream.readBits(4); @@ -346,24 +390,26 @@ function rarReadTables(bstream) { function rarDecodeNumber(bstream, dec) { - var DecodeLen = dec.DecodeLen, DecodePos = dec.DecodePos, DecodeNum = dec.DecodeNum; + var DecodeLen = dec.DecodeLen, + DecodePos = dec.DecodePos, + DecodeNum = dec.DecodeNum; var bitField = bstream.getBits() & 0xfffe; //some sort of rolled out binary search var bits = ((bitField < DecodeLen[8]) ? ((bitField < DecodeLen[4]) ? ((bitField < DecodeLen[2]) ? - ((bitField < DecodeLen[1]) ? 1 : 2) - : ((bitField < DecodeLen[3]) ? 3 : 4)) - : (bitField < DecodeLen[6]) ? - ((bitField < DecodeLen[5]) ? 5 : 6) - : ((bitField < DecodeLen[7]) ? 7 : 8)) - : ((bitField < DecodeLen[12]) ? + ((bitField < DecodeLen[1]) ? 1 : 2) : + ((bitField < DecodeLen[3]) ? 3 : 4)) : + (bitField < DecodeLen[6]) ? + ((bitField < DecodeLen[5]) ? 5 : 6) : + ((bitField < DecodeLen[7]) ? 7 : 8)) : + ((bitField < DecodeLen[12]) ? ((bitField < DecodeLen[10]) ? - ((bitField < DecodeLen[9]) ? 9 : 10) - : ((bitField < DecodeLen[11]) ? 11 : 12)) - : (bitField < DecodeLen[14]) ? - ((bitField < DecodeLen[13]) ? 13 : 14) - : 15)); + ((bitField < DecodeLen[9]) ? 9 : 10) : + ((bitField < DecodeLen[11]) ? 11 : 12)) : + (bitField < DecodeLen[14]) ? + ((bitField < DecodeLen[13]) ? 13 : 14) : + 15)); bstream.readBits(bits); var N = DecodePos[bits] + ((bitField - DecodeLen[bits - 1]) >>> (16 - bits)); @@ -372,12 +418,17 @@ function rarDecodeNumber(bstream, dec) { function rarMakeDecodeTables(BitLength, offset, dec, size) { - var DecodeLen = dec.DecodeLen, DecodePos = dec.DecodePos, DecodeNum = dec.DecodeNum; - var LenCount = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - TmpPos = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - N = 0, M = 0; + var DecodeLen = dec.DecodeLen; + var DecodePos = dec.DecodePos; + var DecodeNum = dec.DecodeNum; + var LenCount = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + var TmpPos = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + var N = 0; + var M = 0; var i; - for (i = DecodeNum.length; i--;) DecodeNum[i] = 0; + for (i = DecodeNum.length; i--;) { + DecodeNum[i] = 0; + } for (i = 0; i < size; i++) { LenCount[BitLength[i + offset] & 0xF]++; } @@ -398,25 +449,28 @@ function rarMakeDecodeTables(BitLength, offset, dec, size) { TmpPos[I] = DecodePos[I]; } for (I = 0; I < size; ++I) { - if (BitLength[I + offset] !== 0) { - DecodeNum[ TmpPos[ BitLength[offset + I] & 0xF ]++] = I; + if (BitLength[I + offset] != 0) { + DecodeNum[TmpPos[BitLength[offset + I] & 0xF]++] = I; } } + } // TODO: implement -function Unpack15() { //bstream, Solid) { +/** + * @param {bitjs.io.BitStream} bstream + * @param {boolean} Solid + */ +function unpack15() { //bstream, Solid) { info("ERROR! RAR 1.5 compression not supported"); } -var lowDistRepCount = 0, prevLowDist = 0; - -var rOldDist = [0, 0, 0, 0]; - -var lastDist = 0; -var lastLength = 0; - -function Unpack20(bstream) { //, Solid) { +/** + * Unpacks the bit stream into rBuffer using the Unpack20 algorithm. + * @param {bitjs.io.BitStream} bstream + * @param {boolean} Solid + */ +function unpack20(bstream) { //, Solid) { var destUnpSize = rBuffer.data.length; var oldDistPtr = 0; var Length; @@ -441,7 +495,9 @@ function Unpack20(bstream) { //, Solid) { } if (Distance >= 0x2000) { Length++; - if (Distance >= 0x40000) Length++; + if (Distance >= 0x40000) { + Length++; + } } lastLength = Length; lastDist = rOldDist[oldDistPtr++ & 3] = Distance; @@ -450,9 +506,7 @@ function Unpack20(bstream) { //, Solid) { } if (num === 269) { rarReadTables20(bstream); - rarUpdateProgress(); - continue; } if (num === 256) { @@ -470,8 +524,10 @@ function Unpack20(bstream) { //, Solid) { if (Distance >= 0x101) { Length++; if (Distance >= 0x2000) { - Length++; - if (Distance >= 0x40000) Length++; + Length++ + if (Distance >= 0x40000) { + Length++; + } } } lastLength = Length; @@ -500,7 +556,6 @@ function rarUpdateProgress() { postProgress(); } - var rNC20 = 298, rDC20 = 48, rRC20 = 28, @@ -516,7 +571,9 @@ function rarReadTables20(bstream) { var i; bstream.readBits(1); if (!bstream.readBits(1)) { - for (i = UnpOldTable20.length; i--;) UnpOldTable20[i] = 0; + for (i = UnpOldTable20.length; i--;) { + UnpOldTable20[i] = 0; + } } TableSize = rNC20 + rDC20 + rRC20; for (I = 0; I < rBC20; I++) { @@ -549,11 +606,254 @@ function rarReadTables20(bstream) { rarMakeDecodeTables(Table, 0, LD, rNC20); rarMakeDecodeTables(Table, rNC20, DD, rDC20); rarMakeDecodeTables(Table, rNC20 + rDC20, RD, rRC20); - for (i = UnpOldTable20.length; i--;) UnpOldTable20[i] = Table[i]; + for (i = UnpOldTable20.length; i--;) { + UnpOldTable20[i] = Table[i]; + } +} + +var lowDistRepCount = 0; +var prevLowDist = 0; + +var rOldDist = [0, 0, 0, 0]; +var lastDist; +var lastLength; + +// ============================================================================================== // + +// Unpack code specific to RarVM +var VM = new RarVM(); + +/** + * Filters code, one entry per filter. + * @type {Array} + */ +var Filters = []; + +/** + * Filters stack, several entrances of same filter are possible. + * @type {Array} + */ +var PrgStack = []; + +/** + * Lengths of preceding blocks, one length per filter. Used to reduce + * size required to write block length if lengths are repeating. + * @type {Array} + */ +var OldFilterLengths = []; + +var LastFilter = 0; + +function InitFilters() { + OldFilterLengths = []; + LastFilter = 0; + Filters = []; + PrgStack = []; +} + + +/** + * @param {number} firstByte The first byte (flags). + * @param {Uint8Array} vmCode An array of bytes. + */ +function rarAddVMCode(firstByte, vmCode) { + VM.init(); + var bstream = new bitjs.io.BitStream(vmCode.buffer, true /* rtl */ ); + + var filtPos; + if (firstByte & 0x80) { + filtPos = RarVM.readData(bstream); + if (filtPos == 0) { + InitFilters(); + } else { + filtPos--; + } + } else { + filtPos = LastFilter; + } + + if (filtPos > Filters.length || filtPos > OldFilterLengths.length) { + return false; + } + + LastFilter = filtPos; + var newFilter = (filtPos == Filters.length); + + // new filter for PrgStack + var stackFilter = new UnpackFilter(); + var filter = null; + // new filter code, never used before since VM reset + if (newFilter) { + // too many different filters, corrupt archive + if (filtPos > 1024) { + return false; + } + + filter = new UnpackFilter(); + Filters.push(filter); + stackFilter.ParentFilter = (Filters.length - 1); + OldFilterLengths.push(0); // OldFilterLengths.Add(1) + filter.ExecCount = 0; + } else { // filter was used in the past + filter = Filters[filtPos]; + stackFilter.ParentFilter = filtPos; + filter.ExecCount++; + } + + var emptyCount = 0; + for (var i = 0; i < PrgStack.length; ++i) { + PrgStack[i - emptyCount] = PrgStack[i]; + + if (PrgStack[i] == null) { + emptyCount++; + } + if (emptyCount > 0) { + PrgStack[i] = null; + } + } + + if (emptyCount == 0) { + PrgStack.push(null); //PrgStack.Add(1); + emptyCount = 1; + } + + var stackPos = PrgStack.length - emptyCount; + PrgStack[stackPos] = stackFilter; + stackFilter.ExecCount = filter.ExecCount; + + var blockStart = RarVM.readData(bstream); + if (firstByte & 0x40) { + blockStart += 258; + } + stackFilter.BlockStart = (blockStart + rBuffer.ptr) & MAXWINMASK; + + if (firstByte & 0x20) { + stackFilter.BlockLength = RarVM.readData(bstream); + } else { + stackFilter.BlockLength = filtPos < OldFilterLengths.length ? + OldFilterLengths[filtPos] : + 0; + } + stackFilter.NextWindow = (wBuffer.ptr != rBuffer.ptr) && + (((wBuffer.ptr - rBuffer.ptr) & MAXWINMASK) <= blockStart); + + OldFilterLengths[filtPos] = stackFilter.BlockLength; + + for (var i = 0; i < 7; ++i) { + stackFilter.Prg.InitR[i] = 0; + } + stackFilter.Prg.InitR[3] = VM_GLOBALMEMADDR; + stackFilter.Prg.InitR[4] = stackFilter.BlockLength; + stackFilter.Prg.InitR[5] = stackFilter.ExecCount; + + // set registers to optional parameters if any + if (firstByte & 0x10) { + var initMask = bstream.readBits(7); + for (var i = 0; i < 7; ++i) { + if (initMask & (1 << i)) { + stackFilter.Prg.InitR[i] = RarVM.readData(bstream); + } + } + } + + if (newFilter) { + var vmCodeSize = RarVM.readData(bstream); + if (vmCodeSize >= 0x10000 || vmCodeSize == 0) { + return false; + } + var vmCode = new Uint8Array(vmCodeSize); + for (var i = 0; i < vmCodeSize; ++i) { + //if (Inp.Overflow(3)) + // return(false); + vmCode[i] = bstream.readBits(8); + } + VM.prepare(vmCode, filter.Prg); + } + stackFilter.Prg.Cmd = filter.Prg.Cmd; + stackFilter.Prg.AltCmd = filter.Prg.Cmd; + + var staticDataSize = filter.Prg.StaticData.length; + if (staticDataSize > 0 && staticDataSize < VM_GLOBALMEMSIZE) { + // read statically defined data contained in DB commands + for (var i = 0; i < staticDataSize; ++i) { + stackFilter.Prg.StaticData[i] = filter.Prg.StaticData[i]; + } + } + + if (stackFilter.Prg.GlobalData.length < VM_FIXEDGLOBALSIZE) { + stackFilter.Prg.GlobalData = new Uint8Array(VM_FIXEDGLOBALSIZE); + } + + var globalData = stackFilter.Prg.GlobalData; + for (var i = 0; i < 7; ++i) { + VM.setLowEndianValue(globalData, stackFilter.Prg.InitR[i], i * 4); + } + + VM.setLowEndianValue(globalData, stackFilter.BlockLength, 0x1c); + VM.setLowEndianValue(globalData, 0, 0x20); + VM.setLowEndianValue(globalData, stackFilter.ExecCount, 0x2c); + for (var i = 0; i < 16; ++i) { + globalData[0x30 + i] = 0; + } + + // put data block passed as parameter if any + if (firstByte & 8) { + //if (Inp.Overflow(3)) + // return(false); + var dataSize = RarVM.readData(bstream); + if (dataSize > (VM_GLOBALMEMSIZE - VM_FIXEDGLOBALSIZE)) { + return (false); + } + + var curSize = stackFilter.Prg.GlobalData.length; + if (curSize < dataSize + VM_FIXEDGLOBALSIZE) { + // Resize global data and update the stackFilter and local variable. + var numBytesToAdd = dataSize + VM_FIXEDGLOBALSIZE - curSize; + var newGlobalData = new Uint8Array(globalData.length + numBytesToAdd); + newGlobalData.set(globalData); + + stackFilter.Prg.GlobalData = newGlobalData; + globalData = newGlobalData; + } + //byte *GlobalData=&StackFilter->Prg.GlobalData[VM_FIXEDGLOBALSIZE]; + for (var i = 0; i < dataSize; ++i) { + //if (Inp.Overflow(3)) + // return(false); + globalData[VM_FIXEDGLOBALSIZE + i] = bstream.readBits(8); + } + } + + return true; } -function Unpack29(bstream) { +/** + * @param {!bitjs.io.BitStream} bstream + */ +function rarReadVMCode(bstream) { + var firstByte = bstream.readBits(8); + var length = (firstByte & 7) + 1; + if (length == 7) { + length = bstream.readBits(8) + 7; + } else if (length == 8) { + length = bstream.readBits(16); + } + + // Read all bytes of VM code into an array. + var vmCode = new Uint8Array(length); + for (var i = 0; i < length; i++) { + // Do something here with checking readbuf. + vmCode[i] = bstream.readBits(8); + } + return RarAddVMCode(firstByte, vmCode); +} + +/** + * Unpacks the bit stream into rBuffer using the Unpack29 algorithm. + * @param {bitjs.io.BitStream} bstream + * @param {boolean} Solid + */ +function unpack29(bstream) { // lazy initialize rDDecode and rDBits var DDecode = new Array(rDC); @@ -577,7 +877,9 @@ function Unpack29(bstream) { lastDist = 0; lastLength = 0; var i; - for (i = UnpOldTable.length; i--;) UnpOldTable[i] = 0; + for (i = UnpOldTable.length; i--;) { + UnpOldTable[i] = 0; + } // read in Huffman tables rarReadTables(bstream); @@ -632,12 +934,15 @@ function Unpack29(bstream) { continue; } if (num === 256) { - if (!rarReadEndOfBlock(bstream)) break; + if (!rarReadEndOfBlock(bstream)) { + break; + } continue; } if (num === 257) { - //console.log("READVMCODE"); - if (!rarReadVMCode(bstream)) break; + if (!rarReadVMCode(bstream)) { + break; + } continue; } if (num === 258) { @@ -674,15 +979,193 @@ function Unpack29(bstream) { rarCopyString(2, Distance); continue; } - } + } // while (true) rarUpdateProgress(); + rarWriteBuf(); } -function rarReadEndOfBlock(bstream) { +/** + * Does stuff to the current byte buffer (rBuffer) based on + * the filters loaded into the RarVM and writes out to wBuffer. + */ +function rarWriteBuf() { + var writeSize = (rBuffer.ptr & MAXWINMASK); + + for (var i = 0; i < PrgStack.length; ++i) { + var flt = PrgStack[i]; + if (flt == null) { + continue; + } + + if (flt.NextWindow) { + flt.NextWindow = false; + continue; + } + + var blockStart = flt.BlockStart; + var blockLength = flt.BlockLength; + + // WrittenBorder = wBuffer.ptr + if (((blockStart - wBuffer.ptr) & MAXWINMASK) < writeSize) { + if (wBuffer.ptr != blockStart) { + // Copy blockStart bytes from rBuffer into wBuffer. + rarWriteArea(wBuffer.ptr, blockStart); + writeSize = (rBuffer.ptr - wBuffer.ptr) & MAXWINMASK; + } + if (blockLength <= writeSize) { + var blockEnd = (blockStart + blockLength) & MAXWINMASK; + if (blockStart < blockEnd || blockEnd == 0) { + VM.setMemory(0, rBuffer.data.subarray(blockStart, blockStart + blockLength), blockLength); + } else { + var firstPartLength = MAXWINSIZE - blockStart; + VM.setMemory(0, rBuffer.data.subarray(blockStart, blockStart + firstPartLength), firstPartLength); + VM.setMemory(firstPartLength, rBuffer.data, blockEnd); + } + + var parentPrg = Filters[flt.ParentFilter].Prg; + var prg = flt.Prg; + + if (parentPrg.GlobalData.length > VM_FIXEDGLOBALSIZE) { + // Copy global data from previous script execution if any. + prg.GlobalData = new Uint8Array(parentPrg.GlobalData); + } + + rarExecuteCode(prg); + + if (prg.GlobalData.length > VM_FIXEDGLOBALSIZE) { + // Save global data for next script execution. + var globalDataLen = prg.GlobalData.length; + if (parentPrg.GlobalData.length < globalDataLen) { + parentPrg.GlobalData = new Uint8Array(globalDataLen); + } + parentPrg.GlobalData.set( + this.mem_.subarray(VM_FIXEDGLOBALSIZE, VM_FIXEDGLOBALSIZE + globalDataLen), + VM_FIXEDGLOBALSIZE); + } else { + parentPrg.GlobalData = new Uint8Array(0); + } + + var filteredData = prg.FilteredData; + + PrgStack[i] = null; + while (i + 1 < PrgStack.length) { + var nextFilter = PrgStack[i + 1]; + if (nextFilter == null || nextFilter.BlockStart != blockStart || + nextFilter.BlockLength != filteredData.length || nextFilter.NextWindow) { + break; + } + + // Apply several filters to same data block. + + VM.setMemory(0, filteredData, filteredData.length); + + var parentPrg = Filters[nextFilter.ParentFilter].Prg; + var nextPrg = nextFilter.Prg; + + var globalDataLen = parentPrg.GlobalData.length; + if (globalDataLen > VM_FIXEDGLOBALSIZE) { + // Copy global data from previous script execution if any. + nextPrg.GlobalData = new Uint8Array(globalDataLen); + nextPrg.GlobalData.set(parentPrg.GlobalData.subarray(VM_FIXEDGLOBALSIZE, VM_FIXEDGLOBALSIZE + globalDataLen), VM_FIXEDGLOBALSIZE); + } + + rarExecuteCode(nextPrg); + + if (nextPrg.GlobalData.length > VM_GLOBALMEMSIZE) { + // Save global data for next script execution. + var globalDataLen = nextPrg.GlobalData.length; + if (parentPrg.GlobalData.length < globalDataLen) { + parentPrg.GlobalData = new Uint8Array(globalDataLen); + } + parentPrg.GlobalData.set( + this.mem_.subarray(VM_FIXEDGLOBALSIZE, VM_FIXEDGLOBALSIZE + globalDataLen), + VM_FIXEDGLOBALSIZE); + } else { + parentPrg.GlobalData = new Uint8Array(0); + } + + filteredData = nextPrg.FilteredData; + i++; + PrgStack[i] = null; + } // while (i + 1 < PrgStack.length) + + for (var j = 0; j < filteredData.length; ++j) { + wBuffer.insertByte(filteredData[j]); + } + writeSize = (rBuffer.ptr - wBuffer.ptr) & MAXWINMASK; + } // if (blockLength <= writeSize) + else { + for (var j = i; j < PrgStack.length; ++j) { + var flt = PrgStack[j]; + if (flt != null && flt.NextWindow) { + flt.NextWindow = false; + } + } + //WrPtr=WrittenBorder; + return; + } + } // if (((blockStart - wBuffer.ptr) & MAXWINMASK) < writeSize) + } // for (var i = 0; i < PrgStack.length; ++i) + + // Write any remaining bytes from rBuffer to wBuffer; + rarWriteArea(wBuffer.ptr, rBuffer.ptr); + + // Now that the filtered buffer has been written, swap it back to rBuffer. + rBuffer = wBuffer; +} + +/** + * Copy bytes from rBuffer to wBuffer. + * @param {number} startPtr The starting point to copy from rBuffer. + * @param {number} endPtr The ending point to copy from rBuffer. + */ +function rarWriteArea(startPtr, endPtr) { + if (endPtr < startPtr) { + console.error('endPtr < startPtr, endPtr=' + endPtr + ', startPtr=' + startPtr); + // rarWriteData(startPtr, -(int)StartPtr & MAXWINMASK); + // RarWriteData(0, endPtr); + return; + } else if (startPtr < endPtr) { + rarWriteData(startPtr, endPtr - startPtr); + } +} + +/** + * Writes bytes into wBuffer from rBuffer. + * @param {number} offset The starting point to copy bytes from rBuffer. + * @param {number} numBytes The number of bytes to copy. + */ +function rarWriteData(offset, numBytes) { + if (wBuffer.ptr >= rBuffer.data.length) { + return; + } + var leftToWrite = rBuffer.data.length - wBuffer.ptr; + if (numBytes > leftToWrite) { + numBytes = leftToWrite; + } + for (var i = 0; i < numBytes; ++i) { + wBuffer.insertByte(rBuffer.data[offset + i]); + } +} + +/** + * @param {VM_PreparedProgram} prg + */ +function rarExecuteCode(prg) { + if (prg.GlobalData.length > 0) { + var writtenFileSize = wBuffer.ptr; + prg.InitR[6] = writtenFileSize; + VM.setLowEndianValue(prg.GlobalData, writtenFileSize, 0x24); + VM.setLowEndianValue(prg.GlobalData, (writtenFileSize >>> 32) >> 0, 0x28); + VM.execute(prg); + } +} +function rarReadEndOfBlock(bstream) { rarUpdateProgress(); - var NewTable = false, NewFile = false; + var NewTable = false, + NewFile = false; if (bstream.readBits(1)) { NewTable = true; } else { @@ -693,31 +1176,6 @@ function rarReadEndOfBlock(bstream) { return !(NewFile || NewTable && !rarReadTables(bstream)); } - -function rarReadVMCode(bstream) { - var FirstByte = bstream.readBits(8); - var Length = (FirstByte & 7) + 1; - if (Length === 7) { - Length = bstream.readBits(8) + 7; - } else if (Length === 8) { - Length = bstream.readBits(16); - } - var vmCode = []; - for (var I = 0; I < Length; I++) { - //do something here with cheking readbuf - vmCode.push(bstream.readBits(8)); - } - return RarAddVMCode(FirstByte, vmCode, Length); -} - -function RarAddVMCode(firstByte, vmCode, length) { - //console.log(vmCode); - if (vmCode.length > 0) { - info("Error! RarVM not supported yet!"); - } - return true; -} - function rarInsertLastMatch(length, distance) { lastDist = distance; lastLength = length; @@ -728,59 +1186,69 @@ function rarInsertOldDist(distance) { rOldDist.splice(0, 0, distance); } -//this is the real function, the other one is for debugging +/** + * Copies len bytes from distance bytes ago in the buffer to the end of the + * current byte buffer. + * @param {number} length How many bytes to copy. + * @param {number} distance How far back in the buffer from the current write + * pointer to start copying from. + */ function rarCopyString(length, distance) { - var destPtr = rBuffer.ptr - distance; - if (destPtr < 0) { + var srcPtr = rBuffer.ptr - distance; + if (srcPtr < 0) { var l = rOldBuffers.length; - while (destPtr < 0) { - destPtr = rOldBuffers[--l].data.length + destPtr; + while (srcPtr < 0) { + srcPtr = rOldBuffers[--l].data.length + srcPtr; + } + // TODO: lets hope that it never needs to read beyond file boundaries + while (length--) { + rBuffer.insertByte(rOldBuffers[l].data[srcPtr++]); } - //TODO: lets hope that it never needs to read beyond file boundaries - while (length--) rBuffer.insertByte(rOldBuffers[l].data[destPtr++]); } if (length > distance) { - while (length--) rBuffer.insertByte(rBuffer.data[destPtr++]); + while (length--) { + rBuffer.insertByte(rBuffer.data[srcPtr++]); + } } else { - rBuffer.insertBytes(rBuffer.data.subarray(destPtr, destPtr + length)); + rBuffer.insertBytes(rBuffer.data.subarray(srcPtr, srcPtr + length)); } } -var rOldBuffers = []; -// v must be a valid RarVolume +/** + * @param {RarLocalFile} v + */ function unpack(v) { - // TODO: implement what happens when unpVer is < 15 - var Ver = v.header.unpVer <= 15 ? 15 : v.header.unpVer, - Solid = v.header.LHD_SOLID, - bstream = new bitjs.io.BitStream(v.fileData.buffer, true /* rtl */, v.fileData.byteOffset, v.fileData.byteLength ); + var Ver = v.header.unpVer <= 15 ? 15 : v.header.unpVer; + var Solid = v.header.LHD_SOLID; + var bstream = new bitjs.io.BitStream(v.fileData.buffer, true /* rtl */ , v.fileData.byteOffset, v.fileData.byteLength); rBuffer = new bitjs.io.ByteBuffer(v.header.unpackedSize); info("Unpacking " + v.filename + " RAR v" + Ver); - switch(Ver) { + switch (Ver) { case 15: // rar 1.5 compression - Unpack15(); //(bstream, Solid); + unpack15(); //(bstream, Solid); break; case 20: // rar 2.x compression case 26: // files larger than 2GB - Unpack20(bstream); //, Solid); + unpack20(bstream); //, Solid); break; case 29: // rar 3.x compression case 36: // alternative hash - Unpack29(bstream); + wBuffer = new bitjs.io.ByteBuffer(rBuffer.data.length); + unpack29(bstream); break; } // switch(method) rOldBuffers.push(rBuffer); - //TODO: clear these old buffers when there's over 4MB of history + // TODO: clear these old buffers when there's over 4MB of history return rBuffer.data; } // bstream is a bit stream var RarLocalFile = function(bstream) { - this.header = new RarVolumeHeader(bstream); this.filename = this.header.filename; @@ -798,7 +1266,6 @@ var RarLocalFile = function(bstream) { }; RarLocalFile.prototype.unrar = function() { - if (!this.header.flags.LHD_SPLIT_BEFORE) { // unstore file if (this.header.method === 0x30) { @@ -836,7 +1303,6 @@ var unrar = function(arrayBuffer) { header.headType === 0x72 && header.flags.value === 0x1A21 && header.headSize === 7) { - info("Found RAR signature"); var mhead = new RarVolumeHeader(bstream); @@ -859,7 +1325,7 @@ var unrar = function(arrayBuffer) { break; } //info("bstream" + bstream.bytePtr+"/"+bstream.bytes.length); - } while ( localFile.isValid ); + } while (localFile.isValid); totalFilesInArchive = localFiles.length; // now we have all information but things are unpacked @@ -870,7 +1336,9 @@ var unrar = function(arrayBuffer) { return aname > bname ? 1 : -1; }); - info(localFiles.map(function(a) {return a.filename;}).join(", ")); + info(localFiles.map(function(a) { + return a.filename; + }).join(", ")); for (var i = 0; i < localFiles.length; ++i) { var localfile = localFiles[i]; diff --git a/cps/static/js/archive/untar.js b/cps/static/js/archive/untar.js new file mode 100644 index 00000000..d6ef3018 --- /dev/null +++ b/cps/static/js/archive/untar.js @@ -0,0 +1,167 @@ +/** + * untar.js + * + * Licensed under the MIT License + * + * Copyright(c) 2011 Google Inc. + * + * Reference Documentation: + * + * TAR format: http://www.gnu.org/software/automake/manual/tar/Standard.html + */ + +// This file expects to be invoked as a Worker (see onmessage below). +importScripts('../io/bytestream.js'); +importScripts('archive.js'); + +// Progress variables. +var currentFilename = ""; +var currentFileNumber = 0; +var currentBytesUnarchivedInFile = 0; +var currentBytesUnarchived = 0; +var totalUncompressedBytesInArchive = 0; +var totalFilesInArchive = 0; + +// Helper functions. +var info = function(str) { + postMessage(new bitjs.archive.UnarchiveInfoEvent(str)); +}; +var err = function(str) { + postMessage(new bitjs.archive.UnarchiveErrorEvent(str)); +}; + +// Removes all characters from the first zero-byte in the string onwards. +var readCleanString = function(bstr, numBytes) { + var str = bstr.readString(numBytes); + var zIndex = str.indexOf(String.fromCharCode(0)); + return zIndex != -1 ? str.substr(0, zIndex) : str; +}; + +var postProgress = function() { + postMessage(new bitjs.archive.UnarchiveProgressEvent( + currentFilename, + currentFileNumber, + currentBytesUnarchivedInFile, + currentBytesUnarchived, + totalUncompressedBytesInArchive, + totalFilesInArchive, + )); +}; + +// takes a ByteStream and parses out the local file information +var TarLocalFile = function(bstream) { + this.isValid = false; + + var bytesRead = 0; + + // Read in the header block + this.name = readCleanString(bstream, 100); + this.mode = readCleanString(bstream, 8); + this.uid = readCleanString(bstream, 8); + this.gid = readCleanString(bstream, 8); + this.size = parseInt(readCleanString(bstream, 12), 8); + this.mtime = readCleanString(bstream, 12); + this.chksum = readCleanString(bstream, 8); + this.typeflag = readCleanString(bstream, 1); + this.linkname = readCleanString(bstream, 100); + this.maybeMagic = readCleanString(bstream, 6); + + if (this.maybeMagic == "ustar") { + this.version = readCleanString(bstream, 2); + this.uname = readCleanString(bstream, 32); + this.gname = readCleanString(bstream, 32); + this.devmajor = readCleanString(bstream, 8); + this.devminor = readCleanString(bstream, 8); + this.prefix = readCleanString(bstream, 155); + + if (this.prefix.length) { + this.name = this.prefix + this.name; + } + bstream.readBytes(12); // 512 - 500 + } else { + bstream.readBytes(255); // 512 - 257 + } + + bytesRead += 512; + + // Done header, now rest of blocks are the file contents. + this.filename = this.name; + this.fileData = null; + + info("Untarring file '" + this.filename + "'"); + info(" size = " + this.size); + info(" typeflag = " + this.typeflag); + + // A regular file. + if (this.typeflag == 0) { + info(" This is a regular file."); + var sizeInBytes = parseInt(this.size); + this.fileData = new Uint8Array(bstream.readBytes(sizeInBytes)); + bytesRead += sizeInBytes; + if (this.name.length > 0 && this.size > 0 && this.fileData && this.fileData.buffer) { + this.isValid = true; + } + + // Round up to 512-byte blocks. + var remaining = 512 - bytesRead % 512; + if (remaining > 0 && remaining < 512) { + bstream.readBytes(remaining); + } + } else if (this.typeflag == 5) { + info(" This is a directory."); + } +} + +var untar = function(arrayBuffer) { + postMessage(new bitjs.archive.UnarchiveStartEvent()); + currentFilename = ""; + currentFileNumber = 0; + currentBytesUnarchivedInFile = 0; + currentBytesUnarchived = 0; + totalUncompressedBytesInArchive = 0; + totalFilesInArchive = 0; + allLocalFiles = []; + + var bstream = new bitjs.io.ByteStream(arrayBuffer); + postProgress(); + // While we don't encounter an empty block, keep making TarLocalFiles. + while (bstream.peekNumber(4) != 0) { + var oneLocalFile = new TarLocalFile(bstream); + if (oneLocalFile && oneLocalFile.isValid) { + // If we make it to this point and haven't thrown an error, we have successfully + // read in the data for a local file, so we can update the actual bytestream. + + allLocalFiles.push(oneLocalFile); + totalUncompressedBytesInArchive += oneLocalFile.size; + + // update progress + currentFilename = oneLocalFile.filename; + currentFileNumber = totalFilesInArchive++; + currentBytesUnarchivedInFile = oneLocalFile.size; + currentBytesUnarchived += oneLocalFile.size; + postMessage(new bitjs.archive.UnarchiveExtractEvent(oneLocalFile)); + postProgress(); + } + } + totalFilesInArchive = allLocalFiles.length; + + postProgress(); + postMessage(new bitjs.archive.UnarchiveFinishEvent()); +}; + +// event.data.file has the first ArrayBuffer. +// event.data.bytes has all subsequent ArrayBuffers. +onmessage = function(event) { + try { + untar(event.data.file, true); + } catch (e) { + if (typeof e === "string" && e.startsWith("Error! Overflowed")) { + // Overrun the buffer. + // unarchiveState = UnarchiveState.WAITING; + } else { + console.error("Found an error while untarring"); + console.log(e); + throw e; + } + } +}; diff --git a/cps/static/js/unzip.js b/cps/static/js/archive/unzip.js similarity index 88% rename from cps/static/js/unzip.js rename to cps/static/js/archive/unzip.js index 0a067516..45cb38ff 100644 --- a/cps/static/js/unzip.js +++ b/cps/static/js/archive/unzip.js @@ -1,6 +1,8 @@ /** * unzip.js * + * Licensed under the MIT License + * * Copyright(c) 2011 Google Inc. * Copyright(c) 2011 antimatter15 * @@ -12,8 +14,10 @@ /* global bitjs, importScripts, Uint8Array*/ // This file expects to be invoked as a Worker (see onmessage below). -importScripts("io.js"); -importScripts("archive.js"); +importScripts('../io/bitstream.js'); +importScripts('../io/bytebuffer.js'); +importScripts('../io/bytestream.js'); +importScripts('archive.js'); // Progress variables. var currentFilename = ""; @@ -47,7 +51,7 @@ var zDigitalSignatureSignature = 0x05054b50; // takes a ByteStream and parses out the local file information var ZipLocalFile = function(bstream) { - if (typeof bstream != typeof {} || !bstream.readNumber || typeof bstream.readNumber != typeof function() {}) { + if (typeof bstream !== typeof {} || !bstream.readNumber || typeof bstream.readNumber !== typeof function() {}) { return null; } @@ -98,7 +102,7 @@ var ZipLocalFile = function(bstream) { // "This descriptor exists only if bit 3 of the general purpose bit flag is set" // But how do you figure out how big the file data is if you don't know the compressedSize // from the header?!? - if ((this.generalPurpose & bitjs.BIT[3]) != 0) { + if ((this.generalPurpose & bitjs.BIT[3]) !== 0) { this.crc32 = bstream.readNumber(4); this.compressedSize = bstream.readNumber(4); this.uncompressedSize = bstream.readNumber(4); @@ -109,14 +113,14 @@ var ZipLocalFile = function(bstream) { ZipLocalFile.prototype.unzip = function() { // Zip Version 1.0, no compression (store only) - if (this.compressionMethod == 0 ) { + if (this.compressionMethod === 0 ) { info("ZIP v" + this.version + ", store only: " + this.filename + " (" + this.compressedSize + " bytes)"); currentBytesUnarchivedInFile = this.compressedSize; currentBytesUnarchived += this.compressedSize; this.fileData = zeroCompression(this.fileData, this.uncompressedSize); } // version == 20, compression method == 8 (DEFLATE) - else if (this.compressionMethod == 8) { + else if (this.compressionMethod === 8) { info("ZIP v2.0, DEFLATE: " + this.filename + " (" + this.compressedSize + " bytes)"); this.fileData = inflate(this.fileData, this.uncompressedSize); } @@ -143,10 +147,10 @@ var unzip = function(arrayBuffer) { var bstream = new bitjs.io.ByteStream(arrayBuffer); // detect local file header signature or return null - if (bstream.peekNumber(4) == zLocalFileHeaderSignature) { + if (bstream.peekNumber(4) === zLocalFileHeaderSignature) { var localFiles = []; // loop until we don't see any more local files - while (bstream.peekNumber(4) == zLocalFileHeaderSignature) { + while (bstream.peekNumber(4) === zLocalFileHeaderSignature) { var oneLocalFile = new ZipLocalFile(bstream); // this should strip out directories/folders if (oneLocalFile && oneLocalFile.uncompressedSize > 0 && oneLocalFile.fileData) { @@ -164,7 +168,7 @@ var unzip = function(arrayBuffer) { }); // archive extra data record - if (bstream.peekNumber(4) == zArchiveExtraDataSignature) { + if (bstream.peekNumber(4) === zArchiveExtraDataSignature) { info(" Found an Archive Extra Data Signature"); // skipping this record for now @@ -175,11 +179,11 @@ var unzip = function(arrayBuffer) { // central directory structure // TODO: handle the rest of the structures (Zip64 stuff) - if (bstream.peekNumber(4) == zCentralFileHeaderSignature) { + if (bstream.peekNumber(4) === zCentralFileHeaderSignature) { info(" Found a Central File Header"); // read all file headers - while (bstream.peekNumber(4) == zCentralFileHeaderSignature) { + while (bstream.peekNumber(4) === zCentralFileHeaderSignature) { bstream.readNumber(4); // signature bstream.readNumber(2); // version made by bstream.readNumber(2); // version needed to extract @@ -205,7 +209,7 @@ var unzip = function(arrayBuffer) { } // digital signature - if (bstream.peekNumber(4) == zDigitalSignatureSignature) { + if (bstream.peekNumber(4) === zDigitalSignatureSignature) { info(" Found a Digital Signature"); bstream.readNumber(4); @@ -230,7 +234,7 @@ var unzip = function(arrayBuffer) { // actually do the unzipping localfile.unzip(); - if (localfile.fileData != null) { + if (localfile.fileData !== null) { postMessage(new bitjs.archive.UnarchiveExtractEvent(localfile)); postProgress(); } @@ -245,7 +249,7 @@ var unzip = function(arrayBuffer) { // containing {length: 6, symbol: X} function getHuffmanCodes(bitLengths) { // ensure bitLengths is an array containing at least one element - if (typeof bitLengths != typeof [] || bitLengths.length < 1) { + if (typeof bitLengths !== typeof [] || bitLengths.length < 1) { err("Error! getHuffmanCodes() called with an invalid array"); return null; } @@ -259,12 +263,12 @@ function getHuffmanCodes(bitLengths) { for (var i = 0; i < numLengths; ++i) { var length = bitLengths[i]; // test to ensure each bit length is a positive, non-zero number - if (typeof length != typeof 1 || length < 0) { + if (typeof length !== typeof 1 || length < 0) { err("bitLengths contained an invalid number in getHuffmanCodes(): " + length + " of type " + (typeof length)); return null; } // increment the appropriate bitlength count - if (blCount[length] == undefined) blCount[length] = 0; + if (typeof blCount[length] === "undefined") blCount[length] = 0; // a length of zero means this symbol is not participating in the huffman coding if (length > 0) blCount[length]++; @@ -275,18 +279,19 @@ function getHuffmanCodes(bitLengths) { var nextCode = [], code = 0; for (var bits = 1; bits <= MAX_BITS; ++bits) { - var length = bits - 1; + var length2 = bits - 1; // ensure undefined lengths are zero - if (blCount[length] == undefined) blCount[length] = 0; + if (typeof blCount[length2] === "undefined") blCount[length2] = 0; code = (code + blCount[bits - 1]) << 1; nextCode [bits] = code; } // Step 3: Assign numerical values to all codes - var table = {}, tableLength = 0; + var table = {}, + tableLength = 0; for (var n = 0; n < numLengths; ++n) { var len = bitLengths[n]; - if (len != 0) { + if (len !== 0) { table[nextCode [len]] = { length: len, symbol: n }; //, bitstring: binaryValueToString(nextCode [len],len) }; tableLength++; nextCode [len]++; @@ -316,6 +321,7 @@ function getHuffmanCodes(bitLengths) { // fixed Huffman codes go from 7-9 bits, so we need an array whose index can hold up to 9 bits var fixedHCtoLiteral = null; var fixedHCtoDistance = null; + function getFixedLiteralTable() { // create once if (!fixedHCtoLiteral) { @@ -331,6 +337,7 @@ function getFixedLiteralTable() { } return fixedHCtoLiteral; } + function getFixedDistanceTable() { // create once if (!fixedHCtoDistance) { @@ -358,7 +365,7 @@ function decodeSymbol(bstream, hcTable) { ++len; // check against Huffman Code table and break if found - if (hcTable.hasOwnProperty(code) && hcTable[code].length == len) { + if (hcTable.hasOwnProperty(code) && hcTable[code].length === len) { break; } @@ -387,16 +394,36 @@ Code Bits Length(s) Code Bits Lengths Code Bits Length(s) 264 0 10 274 3 43-50 284 5 227-257 265 1 11,12 275 3 51-58 285 0 258 266 1 13,14 276 3 59-66 - */ var LengthLookupTable = [ - [0, 3], [0, 4], [0, 5], [0, 6], - [0, 7], [0, 8], [0, 9], [0, 10], - [1, 11], [1, 13], [1, 15], [1, 17], - [2, 19], [2, 23], [2, 27], [2, 31], - [3, 35], [3, 43], [3, 51], [3, 59], - [4, 67], [4, 83], [4, 99], [4, 115], - [5, 131], [5, 163], [5, 195], [5, 227], + [0, 3], + [0, 4], + [0, 5], + [0, 6], + [0, 7], + [0, 8], + [0, 9], + [0, 10], + [1, 11], + [1, 13], + [1, 15], + [1, 17], + [2, 19], + [2, 23], + [2, 27], + [2, 31], + [3, 35], + [3, 43], + [3, 51], + [3, 59], + [4, 67], + [4, 83], + [4, 99], + [4, 115], + [5, 131], + [5, 163], + [5, 195], + [5, 227], [0, 258] ]; /* @@ -415,20 +442,36 @@ var LengthLookupTable = [ 9 3 25-32 19 8 769-1024 29 13 24577-32768 */ var DistLookupTable = [ - [0, 1], [0, 2], [0, 3], [0, 4], - [1, 5], [1, 7], - [2, 9], [2, 13], - [3, 17], [3, 25], - [4, 33], [4, 49], - [5, 65], [5, 97], - [6, 129], [6, 193], - [7, 257], [7, 385], - [8, 513], [8, 769], - [9, 1025], [9, 1537], - [10, 2049], [10, 3073], - [11, 4097], [11, 6145], - [12, 8193], [12, 12289], - [13, 16385], [13, 24577] + [0, 1], + [0, 2], + [0, 3], + [0, 4], + [1, 5], + [1, 7], + [2, 9], + [2, 13], + [3, 17], + [3, 25], + [4, 33], + [4, 49], + [5, 65], + [5, 97], + [6, 129], + [6, 193], + [7, 257], + [7, 385], + [8, 513], + [8, 769], + [9, 1025], + [9, 1537], + [10, 2049], + [10, 3073], + [11, 4097], + [11, 6145], + [12, 8193], + [12, 12289], + [13, 16385], + [13, 24577] ]; function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) { @@ -457,7 +500,7 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) { } else { // end of block reached - if (symbol == 256) { + if (symbol === 256) { break; } else { @@ -485,9 +528,8 @@ function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) { buffer.insertByte(data[ch++]); } } else { - buffer.insertBytes(buffer.data.subarray(ch, ch + length)); + buffer.insertBytes(buffer.data.subarray(ch, ch + length)); } - } // length-distance pair } // length-distance pair or end-of-block } // loop until we reach end of block @@ -514,37 +556,38 @@ function inflate(compressedData, numDecompressedBytes) { compressedData.byteOffset, compressedData.byteLength); var buffer = new bitjs.io.ByteBuffer(numDecompressedBytes); - var numBlocks = 0, blockSize = 0; + var blockSize = 0; // block format: http://tools.ietf.org/html/rfc1951#page-9 + var bFinal = 0; do { - var bFinal = bstream.readBits(1), - bType = bstream.readBits(2); + bFinal = bstream.readBits(1); + var bType = bstream.readBits(2); blockSize = 0; - ++numBlocks; + // ++numBlocks; // no compression if (bType == 0) { // skip remaining bits in this byte - while (bstream.bitPtr != 0) bstream.readBits(1); + while (bstream.bitPtr !== 0) bstream.readBits(1); var len = bstream.readBits(16); bstream.readBits(16); - // TODO: check if nlen is the ones-complement of len? + // TODO: check if nlen is the ones-complement of len? if (len > 0) buffer.insertBytes(bstream.readBytes(len)); blockSize = len; } // fixed Huffman codes - else if(bType == 1) { + else if (bType === 1) { blockSize = inflateBlockData(bstream, getFixedLiteralTable(), getFixedDistanceTable(), buffer); } // dynamic Huffman codes - else if(bType == 2) { + else if (bType === 2) { var numLiteralLengthCodes = bstream.readBits(5) + 257; var numDistanceCodes = bstream.readBits(5) + 1, numCodeLengthCodes = bstream.readBits(4) + 4; // populate the array of code length codes (first de-compaction) - var codeLengthsCodeLengths = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]; + var codeLengthsCodeLengths = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; for (var i = 0; i < numCodeLengthCodes; ++i) { codeLengthsCodeLengths[ CodeLengthCodeOrder[i] ] = bstream.readBits(3); } @@ -575,19 +618,17 @@ function inflate(compressedData, numDecompressedBytes) { if (symbol <= 15) { literalCodeLengths.push(symbol); prevCodeLength = symbol; - } - else if (symbol == 16) { + } else if (symbol === 16) { var repeat = bstream.readBits(2) + 3; while (repeat--) { literalCodeLengths.push(prevCodeLength); } - } - else if (symbol == 17) { + } else if (symbol === 17) { var repeat1 = bstream.readBits(3) + 3; while (repeat1--) { literalCodeLengths.push(0); } - } else if (symbol == 18) { + } else if (symbol === 18) { var repeat2 = bstream.readBits(7) + 11; while (repeat2--) { literalCodeLengths.push(0); @@ -613,7 +654,7 @@ function inflate(compressedData, numDecompressedBytes) { currentBytesUnarchived += blockSize; postProgress(); - } while (bFinal != 1); + } while (bFinal !== 1); // we are done reading blocks if the bFinal bit was set for this block // return the buffer data bytes @@ -623,4 +664,4 @@ function inflate(compressedData, numDecompressedBytes) { // event.data.file has the ArrayBuffer. onmessage = function(event) { unzip(event.data.file, true); -}; +}; \ No newline at end of file diff --git a/cps/static/js/bytestream.js b/cps/static/js/bytestream.js deleted file mode 100644 index cb5df363..00000000 --- a/cps/static/js/bytestream.js +++ /dev/null @@ -1,308 +0,0 @@ -/* - * bytestream.js - * - * Provides readers for byte streams. - * - * Licensed under the MIT License - * - * Copyright(c) 2011 Google Inc. - * Copyright(c) 2011 antimatter15 - */ - -var bitjs = bitjs || {}; -bitjs.io = bitjs.io || {}; - - -/** - * This object allows you to peek and consume bytes as numbers and strings out - * of a stream. More bytes can be pushed into the back of the stream via the - * push() method. - */ -bitjs.io.ByteStream = class { - /** - * @param {ArrayBuffer} ab The ArrayBuffer object. - * @param {number=} opt_offset The offset into the ArrayBuffer - * @param {number=} opt_length The length of this BitStream - */ - constructor(ab, opt_offset, opt_length) { - if (!(ab instanceof ArrayBuffer)) { - throw 'Error! BitArray constructed with an invalid ArrayBuffer object'; - } - - const offset = opt_offset || 0; - const length = opt_length || ab.byteLength; - - /** - * The current page of bytes in the stream. - * @type {Uint8Array} - * @private - */ - this.bytes = new Uint8Array(ab, offset, length); - - /** - * The next pages of bytes in the stream. - * @type {Array} - * @private - */ - this.pages_ = []; - - /** - * The byte in the current page that we will read next. - * @type {Number} - * @private - */ - this.ptr = 0; - - /** - * An ever-increasing number. - * @type {Number} - * @private - */ - this.bytesRead_ = 0; - } - - /** - * Returns how many bytes have been read in the stream since the beginning of time. - */ - getNumBytesRead() { - return this.bytesRead_; - } - - /** - * Returns how many bytes are currently in the stream left to be read. - */ - getNumBytesLeft() { - const bytesInCurrentPage = (this.bytes.byteLength - this.ptr); - return this.pages_.reduce((acc, arr) => acc + arr.length, bytesInCurrentPage); - } - - /** - * Move the pointer ahead n bytes. If the pointer is at the end of the current array - * of bytes and we have another page of bytes, point at the new page. This is a private - * method, no validation is done. - * @param {number} n Number of bytes to increment. - * @private - */ - movePointer_(n) { - this.ptr += n; - this.bytesRead_ += n; - while (this.ptr >= this.bytes.length && this.pages_.length > 0) { - this.ptr -= this.bytes.length; - this.bytes = this.pages_.shift(); - } - } - - /** - * Peeks at the next n bytes as an unsigned number but does not advance the - * pointer. - * @param {number} n The number of bytes to peek at. Must be a positive integer. - * @return {number} The n bytes interpreted as an unsigned number. - */ - peekNumber(n) { - const num = parseInt(n, 10); - if (n !== num || num < 0) { - throw 'Error! Called peekNumber() with a non-positive integer'; - } else if (num === 0) { - return 0; - } - - if (n > 4) { - throw 'Error! Called peekNumber(' + n + - ') but this method can only reliably read numbers up to 4 bytes long'; - } - - if (this.getNumBytesLeft() < num) { - throw 'Error! Overflowed the byte stream while peekNumber()! n=' + num + - ', ptr=' + this.ptr + ', bytes.length=' + this.getNumBytesLeft(); - } - - let result = 0; - let curPage = this.bytes; - let pageIndex = 0; - let ptr = this.ptr; - for (let i = 0; i < num; ++i) { - result |= (curPage[ptr++] << (i * 8)); - - if (ptr >= curPage.length) { - curPage = this.pages_[pageIndex++]; - ptr = 0; - } - } - - return result; - } - - - /** - * Returns the next n bytes as an unsigned number (or -1 on error) - * and advances the stream pointer n bytes. - * @param {number} n The number of bytes to read. Must be a positive integer. - * @return {number} The n bytes interpreted as an unsigned number. - */ - readNumber(n) { - const num = this.peekNumber(n); - this.movePointer_(n); - return num; - } - - - /** - * Returns the next n bytes as a signed number but does not advance the - * pointer. - * @param {number} n The number of bytes to read. Must be a positive integer. - * @return {number} The bytes interpreted as a signed number. - */ - peekSignedNumber(n) { - let num = this.peekNumber(n); - const HALF = Math.pow(2, (n * 8) - 1); - const FULL = HALF * 2; - - if (num >= HALF) num -= FULL; - - return num; - } - - - /** - * Returns the next n bytes as a signed number and advances the stream pointer. - * @param {number} n The number of bytes to read. Must be a positive integer. - * @return {number} The bytes interpreted as a signed number. - */ - readSignedNumber(n) { - const num = this.peekSignedNumber(n); - this.movePointer_(n); - return num; - } - - - /** - * This returns n bytes as a sub-array, advancing the pointer if movePointers - * is true. - * @param {number} n The number of bytes to read. Must be a positive integer. - * @param {boolean} movePointers Whether to move the pointers. - * @return {Uint8Array} The subarray. - */ - peekBytes(n, movePointers) { - const num = parseInt(n, 10); - if (n !== num || num < 0) { - throw 'Error! Called peekBytes() with a non-positive integer'; - } else if (num === 0) { - return new Uint8Array(); - } - - const totalBytesLeft = this.getNumBytesLeft(); - if (num > totalBytesLeft) { - throw 'Error! Overflowed the byte stream during peekBytes! n=' + num + - ', ptr=' + this.ptr + ', bytes.length=' + this.getNumBytesLeft(); - } - - const result = new Uint8Array(num); - let curPage = this.bytes; - let ptr = this.ptr; - let bytesLeftToCopy = num; - let pageIndex = 0; - while (bytesLeftToCopy > 0) { - const bytesLeftInPage = curPage.length - ptr; - const sourceLength = Math.min(bytesLeftToCopy, bytesLeftInPage); - - result.set(curPage.subarray(ptr, ptr + sourceLength), num - bytesLeftToCopy); - - ptr += sourceLength; - if (ptr >= curPage.length) { - curPage = this.pages_[pageIndex++]; - ptr = 0; - } - - bytesLeftToCopy -= sourceLength; - } - - if (movePointers) { - this.movePointer_(num); - } - - return result; - } - - /** - * Reads the next n bytes as a sub-array. - * @param {number} n The number of bytes to read. Must be a positive integer. - * @return {Uint8Array} The subarray. - */ - readBytes(n) { - return this.peekBytes(n, true); - } - - /** - * Peeks at the next n bytes as an ASCII string but does not advance the pointer. - * @param {number} n The number of bytes to peek at. Must be a positive integer. - * @return {string} The next n bytes as a string. - */ - peekString(n) { - const num = parseInt(n, 10); - if (n !== num || num < 0) { - throw 'Error! Called peekString() with a non-positive integer'; - } else if (num === 0) { - return ''; - } - - const totalBytesLeft = this.getNumBytesLeft(); - if (num > totalBytesLeft) { - throw 'Error! Overflowed the byte stream while peekString()! n=' + num + - ', ptr=' + this.ptr + ', bytes.length=' + this.getNumBytesLeft(); - } - - let result = new Array(num); - let curPage = this.bytes; - let pageIndex = 0; - let ptr = this.ptr; - for (let i = 0; i < num; ++i) { - result[i] = String.fromCharCode(curPage[ptr++]); - if (ptr >= curPage.length) { - curPage = this.pages_[pageIndex++]; - ptr = 0; - } - } - - return result.join(''); - } - - /** - * Returns the next n bytes as an ASCII string and advances the stream pointer - * n bytes. - * @param {number} n The number of bytes to read. Must be a positive integer. - * @return {string} The next n bytes as a string. - */ - readString(n) { - const strToReturn = this.peekString(n); - this.movePointer_(n); - return strToReturn; - } - - /** - * Feeds more bytes into the back of the stream. - * @param {ArrayBuffer} ab - */ - push(ab) { - if (!(ab instanceof ArrayBuffer)) { - throw 'Error! ByteStream.push() called with an invalid ArrayBuffer object'; - } - - this.pages_.push(new Uint8Array(ab)); - // If the pointer is at the end of the current page of bytes, this will advance - // to the next page. - this.movePointer_(0); - } - - /** - * Creates a new ByteStream from this ByteStream that can be read / peeked. - * @return {bitjs.io.ByteStream} A clone of this ByteStream. - */ - tee() { - const clone = new bitjs.io.ByteStream(this.bytes.buffer); - clone.bytes = this.bytes; - clone.ptr = this.ptr; - clone.pages_ = this.pages_.slice(); - clone.bytesRead_ = this.bytesRead_; - return clone; - } -} diff --git a/cps/static/js/io/bitstream.js b/cps/static/js/io/bitstream.js new file mode 100644 index 00000000..77ba260f --- /dev/null +++ b/cps/static/js/io/bitstream.js @@ -0,0 +1,233 @@ +/* + * bitstream.js + * + * Provides readers for bitstreams. + * + * Licensed under the MIT License + * + * Copyright(c) 2011 Google Inc. + * Copyright(c) 2011 antimatter15 + */ +var bitjs = bitjs || {}; +bitjs.io = bitjs.io || {}; + +(function() { + + // mask for getting the Nth bit (zero-based) + bitjs.BIT = [0x01, 0x02, 0x04, 0x08, + 0x10, 0x20, 0x40, 0x80, + 0x100, 0x200, 0x400, 0x800, + 0x1000, 0x2000, 0x4000, 0x8000 + ]; + + // mask for getting N number of bits (0-8) + var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF]; + + + /** + * This bit stream peeks and consumes bits out of a binary stream. + * + * @param {ArrayBuffer} ab An ArrayBuffer object or a Uint8Array. + * @param {boolean} rtl Whether the stream reads bits from the byte starting + * from bit 7 to 0 (true) or bit 0 to 7 (false). + * @param {Number} opt_offset The offset into the ArrayBuffer + * @param {Number} opt_length The length of this BitStream + */ + bitjs.io.BitStream = function(ab, rtl, opt_offset, opt_length) { + if (!ab || !ab.toString || ab.toString() !== "[object ArrayBuffer]") { + throw "Error! BitArray constructed with an invalid ArrayBuffer object"; + } + + var offset = opt_offset || 0; + var length = opt_length || ab.byteLength; + this.bytes = new Uint8Array(ab, offset, length); + this.bytePtr = 0; // tracks which byte we are on + this.bitPtr = 0; // tracks which bit we are on (can have values 0 through 7) + this.peekBits = rtl ? this.peekBits_rtl : this.peekBits_ltr; + }; + + + /** + * byte0 byte1 byte2 byte3 + * 7......0 | 7......0 | 7......0 | 7......0 + * + * The bit pointer starts at bit0 of byte0 and moves left until it reaches + * bit7 of byte0, then jumps to bit0 of byte1, etc. + * @param {number} n The number of bits to peek. + * @param {boolean=} movePointers Whether to move the pointer, defaults false. + * @return {number} The peeked bits, as an unsigned number. + */ + bitjs.io.BitStream.prototype.peekBits_ltr = function(n, movePointers) { + if (n <= 0 || typeof n != typeof 1) { + return 0; + } + + var movePointers = movePointers || false, + bytePtr = this.bytePtr, + bitPtr = this.bitPtr, + result = 0, + bitsIn = 0, + bytes = this.bytes; + + // keep going until we have no more bits left to peek at + // TODO: Consider putting all bits from bytes we will need into a variable and then + // shifting/masking it to just extract the bits we want. + // This could be considerably faster when reading more than 3 or 4 bits at a time. + while (n > 0) { + if (bytePtr >= bytes.length) { + throw "Error! Overflowed the bit stream! n=" + n + ", bytePtr=" + bytePtr + ", bytes.length=" + + bytes.length + ", bitPtr=" + bitPtr; + return -1; + } + + var numBitsLeftInThisByte = (8 - bitPtr); + if (n >= numBitsLeftInThisByte) { + var mask = (BITMASK[numBitsLeftInThisByte] << bitPtr); + result |= (((bytes[bytePtr] & mask) >> bitPtr) << bitsIn); + + bytePtr++; + bitPtr = 0; + bitsIn += numBitsLeftInThisByte; + n -= numBitsLeftInThisByte; + } else { + var mask = (BITMASK[n] << bitPtr); + result |= (((bytes[bytePtr] & mask) >> bitPtr) << bitsIn); + + bitPtr += n; + bitsIn += n; + n = 0; + } + } + + if (movePointers) { + this.bitPtr = bitPtr; + this.bytePtr = bytePtr; + } + + return result; + }; + + + /** + * byte0 byte1 byte2 byte3 + * 7......0 | 7......0 | 7......0 | 7......0 + * + * The bit pointer starts at bit7 of byte0 and moves right until it reaches + * bit0 of byte0, then goes to bit7 of byte1, etc. + * @param {number} n The number of bits to peek. + * @param {boolean=} movePointers Whether to move the pointer, defaults false. + * @return {number} The peeked bits, as an unsigned number. + */ + bitjs.io.BitStream.prototype.peekBits_rtl = function(n, movePointers) { + if (n <= 0 || typeof n != typeof 1) { + return 0; + } + + var movePointers = movePointers || false, + bytePtr = this.bytePtr, + bitPtr = this.bitPtr, + result = 0, + bytes = this.bytes; + + // keep going until we have no more bits left to peek at + // TODO: Consider putting all bits from bytes we will need into a variable and then + // shifting/masking it to just extract the bits we want. + // This could be considerably faster when reading more than 3 or 4 bits at a time. + while (n > 0) { + + if (bytePtr >= bytes.length) { + throw "Error! Overflowed the bit stream! n=" + n + ", bytePtr=" + bytePtr + ", bytes.length=" + + bytes.length + ", bitPtr=" + bitPtr; + return -1; + } + + var numBitsLeftInThisByte = (8 - bitPtr); + if (n >= numBitsLeftInThisByte) { + result <<= numBitsLeftInThisByte; + result |= (BITMASK[numBitsLeftInThisByte] & bytes[bytePtr]); + bytePtr++; + bitPtr = 0; + n -= numBitsLeftInThisByte; + } else { + result <<= n; + result |= ((bytes[bytePtr] & (BITMASK[n] << (8 - n - bitPtr))) >> (8 - n - bitPtr)); + + bitPtr += n; + n = 0; + } + } + + if (movePointers) { + this.bitPtr = bitPtr; + this.bytePtr = bytePtr; + } + + return result; + }; + + + /** + * Peek at 16 bits from current position in the buffer. + * Bit at (bytePtr,bitPtr) has the highest position in returning data. + * Taken from getbits.hpp in unrar. + * TODO: Move this out of BitStream and into unrar. + */ + bitjs.io.BitStream.prototype.getBits = function() { + return (((((this.bytes[this.bytePtr] & 0xff) << 16) + + ((this.bytes[this.bytePtr + 1] & 0xff) << 8) + + ((this.bytes[this.bytePtr + 2] & 0xff))) >>> (8 - this.bitPtr)) & 0xffff); + }; + + + /** + * Reads n bits out of the stream, consuming them (moving the bit pointer). + * @param {number} n The number of bits to read. + * @return {number} The read bits, as an unsigned number. + */ + bitjs.io.BitStream.prototype.readBits = function(n) { + return this.peekBits(n, true); + }; + + + /** + * This returns n bytes as a sub-array, advancing the pointer if movePointers + * is true. Only use this for uncompressed blocks as this throws away remaining + * bits in the current byte. + * @param {number} n The number of bytes to peek. + * @param {boolean=} movePointers Whether to move the pointer, defaults false. + * @return {Uint8Array} The subarray. + */ + bitjs.io.BitStream.prototype.peekBytes = function(n, movePointers) { + if (n <= 0 || typeof n != typeof 1) { + return 0; + } + + // from http://tools.ietf.org/html/rfc1951#page-11 + // "Any bits of input up to the next byte boundary are ignored." + while (this.bitPtr != 0) { + this.readBits(1); + } + + var movePointers = movePointers || false; + var bytePtr = this.bytePtr, + bitPtr = this.bitPtr; + + var result = this.bytes.subarray(bytePtr, bytePtr + n); + + if (movePointers) { + this.bytePtr += n; + } + + return result; + }; + + + /** + * @param {number} n The number of bytes to read. + * @return {Uint8Array} The subarray. + */ + bitjs.io.BitStream.prototype.readBytes = function(n) { + return this.peekBytes(n, true); + }; + +})(); \ No newline at end of file diff --git a/cps/static/js/io/bytebuffer.js b/cps/static/js/io/bytebuffer.js new file mode 100644 index 00000000..cb9f955e --- /dev/null +++ b/cps/static/js/io/bytebuffer.js @@ -0,0 +1,121 @@ +/* + * bytestream.js + * + * Provides a writer for bytes. + * + * Licensed under the MIT License + * + * Copyright(c) 2011 Google Inc. + * Copyright(c) 2011 antimatter15 + */ +var bitjs = bitjs || {}; +bitjs.io = bitjs.io || {}; + +(function() { + + + /** + * A write-only Byte buffer which uses a Uint8 Typed Array as a backing store. + * @param {number} numBytes The number of bytes to allocate. + * @constructor + */ + bitjs.io.ByteBuffer = function(numBytes) { + if (typeof numBytes != typeof 1 || numBytes <= 0) { + throw "Error! ByteBuffer initialized with '" + numBytes + "'"; + } + this.data = new Uint8Array(numBytes); + this.ptr = 0; + }; + + + /** + * @param {number} b The byte to insert. + */ + bitjs.io.ByteBuffer.prototype.insertByte = function(b) { + // TODO: throw if byte is invalid? + this.data[this.ptr++] = b; + }; + + + /** + * @param {Array.|Uint8Array|Int8Array} bytes The bytes to insert. + */ + bitjs.io.ByteBuffer.prototype.insertBytes = function(bytes) { + // TODO: throw if bytes is invalid? + this.data.set(bytes, this.ptr); + this.ptr += bytes.length; + }; + + + /** + * Writes an unsigned number into the next n bytes. If the number is too large + * to fit into n bytes or is negative, an error is thrown. + * @param {number} num The unsigned number to write. + * @param {number} numBytes The number of bytes to write the number into. + */ + bitjs.io.ByteBuffer.prototype.writeNumber = function(num, numBytes) { + if (numBytes < 1) { + throw 'Trying to write into too few bytes: ' + numBytes; + } + if (num < 0) { + throw 'Trying to write a negative number (' + num + + ') as an unsigned number to an ArrayBuffer'; + } + if (num > (Math.pow(2, numBytes * 8) - 1)) { + throw 'Trying to write ' + num + ' into only ' + numBytes + ' bytes'; + } + + // Roll 8-bits at a time into an array of bytes. + var bytes = []; + while (numBytes-- > 0) { + var eightBits = num & 255; + bytes.push(eightBits); + num >>= 8; + } + + this.insertBytes(bytes); + }; + + + /** + * Writes a signed number into the next n bytes. If the number is too large + * to fit into n bytes, an error is thrown. + * @param {number} num The signed number to write. + * @param {number} numBytes The number of bytes to write the number into. + */ + bitjs.io.ByteBuffer.prototype.writeSignedNumber = function(num, numBytes) { + if (numBytes < 1) { + throw 'Trying to write into too few bytes: ' + numBytes; + } + + var HALF = Math.pow(2, (numBytes * 8) - 1); + if (num >= HALF || num < -HALF) { + throw 'Trying to write ' + num + ' into only ' + numBytes + ' bytes'; + } + + // Roll 8-bits at a time into an array of bytes. + var bytes = []; + while (numBytes-- > 0) { + var eightBits = num & 255; + bytes.push(eightBits); + num >>= 8; + } + + this.insertBytes(bytes); + }; + + + /** + * @param {string} str The ASCII string to write. + */ + bitjs.io.ByteBuffer.prototype.writeASCIIString = function(str) { + for (var i = 0; i < str.length; ++i) { + var curByte = str.charCodeAt(i); + if (curByte < 0 || curByte > 255) { + throw 'Trying to write a non-ASCII string!'; + } + this.insertByte(curByte); + } + }; + +})(); \ No newline at end of file diff --git a/cps/static/js/io/bytestream.js b/cps/static/js/io/bytestream.js new file mode 100644 index 00000000..066cc5e8 --- /dev/null +++ b/cps/static/js/io/bytestream.js @@ -0,0 +1,162 @@ +/* + * bytestream.js + * + * Provides readers for byte streams. + * + * Licensed under the MIT License + * + * Copyright(c) 2011 Google Inc. + * Copyright(c) 2011 antimatter15 + */ +var bitjs = bitjs || {}; +bitjs.io = bitjs.io || {}; + +(function() { + + + /** + * This object allows you to peek and consume bytes as numbers and strings + * out of an ArrayBuffer. In this buffer, everything must be byte-aligned. + * + * @param {ArrayBuffer} ab The ArrayBuffer object. + * @param {number=} opt_offset The offset into the ArrayBuffer + * @param {number=} opt_length The length of this BitStream + * @constructor + */ + bitjs.io.ByteStream = function(ab, opt_offset, opt_length) { + var offset = opt_offset || 0; + var length = opt_length || ab.byteLength; + this.bytes = new Uint8Array(ab, offset, length); + this.ptr = 0; + }; + + + /** + * Peeks at the next n bytes as an unsigned number but does not advance the + * pointer + * TODO: This apparently cannot read more than 4 bytes as a number? + * @param {number} n The number of bytes to peek at. + * @return {number} The n bytes interpreted as an unsigned number. + */ + bitjs.io.ByteStream.prototype.peekNumber = function(n) { + // TODO: return error if n would go past the end of the stream? + if (n <= 0 || typeof n != typeof 1) + return -1; + + var result = 0; + // read from last byte to first byte and roll them in + var curByte = this.ptr + n - 1; + while (curByte >= this.ptr) { + result <<= 8; + result |= this.bytes[curByte]; + --curByte; + } + return result; + }; + + + /** + * Returns the next n bytes as an unsigned number (or -1 on error) + * and advances the stream pointer n bytes. + * @param {number} n The number of bytes to read. + * @return {number} The n bytes interpreted as an unsigned number. + */ + bitjs.io.ByteStream.prototype.readNumber = function(n) { + var num = this.peekNumber(n); + this.ptr += n; + return num; + }; + + + /** + * Returns the next n bytes as a signed number but does not advance the + * pointer. + * @param {number} n The number of bytes to read. + * @return {number} The bytes interpreted as a signed number. + */ + bitjs.io.ByteStream.prototype.peekSignedNumber = function(n) { + var num = this.peekNumber(n); + var HALF = Math.pow(2, (n * 8) - 1); + var FULL = HALF * 2; + + if (num >= HALF) num -= FULL; + + return num; + }; + + + /** + * Returns the next n bytes as a signed number and advances the stream pointer. + * @param {number} n The number of bytes to read. + * @return {number} The bytes interpreted as a signed number. + */ + bitjs.io.ByteStream.prototype.readSignedNumber = function(n) { + var num = this.peekSignedNumber(n); + this.ptr += n; + return num; + }; + + + /** + * This returns n bytes as a sub-array, advancing the pointer if movePointers + * is true. + * @param {number} n The number of bytes to read. + * @param {boolean} movePointers Whether to move the pointers. + * @return {Uint8Array} The subarray. + */ + bitjs.io.ByteStream.prototype.peekBytes = function(n, movePointers) { + if (n <= 0 || typeof n != typeof 1) { + return null; + } + + var result = this.bytes.subarray(this.ptr, this.ptr + n); + + if (movePointers) { + this.ptr += n; + } + + return result; + }; + + + /** + * Reads the next n bytes as a sub-array. + * @param {number} n The number of bytes to read. + * @return {Uint8Array} The subarray. + */ + bitjs.io.ByteStream.prototype.readBytes = function(n) { + return this.peekBytes(n, true); + }; + + + /** + * Peeks at the next n bytes as a string but does not advance the pointer. + * @param {number} n The number of bytes to peek at. + * @return {string} The next n bytes as a string. + */ + bitjs.io.ByteStream.prototype.peekString = function(n) { + if (n <= 0 || typeof n != typeof 1) { + return ""; + } + + var result = ""; + for (var p = this.ptr, end = this.ptr + n; p < end; ++p) { + result += String.fromCharCode(this.bytes[p]); + } + return result; + }; + + + /** + * Returns the next n bytes as an ASCII string and advances the stream pointer + * n bytes. + * @param {number} n The number of bytes to read. + * @return {string} The next n bytes as a string. + */ + bitjs.io.ByteStream.prototype.readString = function(n) { + var strToReturn = this.peekString(n); + this.ptr += n; + return strToReturn; + }; + +})(); diff --git a/cps/static/js/kthoom.js b/cps/static/js/kthoom.js index d7d97369..9146d603 100644 --- a/cps/static/js/kthoom.js +++ b/cps/static/js/kthoom.js @@ -16,7 +16,6 @@ */ /* global screenfull, bitjs */ - if (window.opera) { window.console.log = function(str) { opera.postError(str); @@ -160,11 +159,16 @@ function initProgressClick() { function loadFromArrayBuffer(ab) { var start = (new Date).getTime(); var h = new Uint8Array(ab, 0, 10); - var pathToBitJS = "../../static/js/"; + var pathToBitJS = "../../static/js/archive/"; if (h[0] === 0x52 && h[1] === 0x61 && h[2] === 0x72 && h[3] === 0x21) { //Rar! unarchiver = new bitjs.archive.Unrarrer(ab, pathToBitJS); } else if (h[0] === 80 && h[1] === 75) { //PK (Zip) unarchiver = new bitjs.archive.Unzipper(ab, pathToBitJS); + } else if (h[0] == 255 && h[1] == 216) { // JPEG + // ToDo: check + updateProgress(100); + lastCompletion = 100; + return; } else { // Try with tar unarchiver = new bitjs.archive.Untarrer(ab, pathToBitJS); } @@ -179,6 +183,10 @@ function loadFromArrayBuffer(ab) { updateProgress(percentage *100); lastCompletion = percentage * 100; }); + unarchiver.addEventListener(bitjs.archive.UnarchiveEvent.Type.INFO, + function(e) { + // console.log(e.msg); 77 Enable debug output here + }); unarchiver.addEventListener(bitjs.archive.UnarchiveEvent.Type.EXTRACT, function(e) { // convert DecompressedFile into a bunch of ImageFiles diff --git a/cps/static/js/untar.js b/cps/static/js/untar.js deleted file mode 100644 index ae6a206c..00000000 --- a/cps/static/js/untar.js +++ /dev/null @@ -1,209 +0,0 @@ -/** -* untar.js -* -* Licensed under the MIT License -* -* Copyright(c) 2011 Google Inc. -* -* Reference Documentation: -* -* TAR format: http://www.gnu.org/software/automake/manual/tar/Standard.html -*/ - -// This file expects to be invoked as a Worker (see onmessage below). -importScripts('bytestream.js'); -importScripts('archive.js'); - -const UnarchiveState = { - NOT_STARTED: 0, - UNARCHIVING: 1, - WAITING: 2, - FINISHED: 3, -}; - -// State - consider putting these into a class. -let unarchiveState = UnarchiveState.NOT_STARTED; -let bytestream = null; -let allLocalFiles = null; -let logToConsole = false; - -// Progress variables. -let currentFilename = ""; -let currentFileNumber = 0; -let currentBytesUnarchivedInFile = 0; -let currentBytesUnarchived = 0; -let totalUncompressedBytesInArchive = 0; -let totalFilesInArchive = 0; - -// Helper functions. -const info = function(str) { - postMessage(new bitjs.archive.UnarchiveInfoEvent(str)); -}; -const err = function(str) { - postMessage(new bitjs.archive.UnarchiveErrorEvent(str)); -}; - -// Removes all characters from the first zero-byte in the string onwards. -var readCleanString = function(bstr, numBytes) { - var str = bstr.readString(numBytes); - var zIndex = str.indexOf(String.fromCharCode(0)); - return zIndex != -1 ? str.substr(0, zIndex) : str; -}; - - -const postProgress = function() { - postMessage(new bitjs.archive.UnarchiveProgressEvent( - currentFilename, - currentFileNumber, - currentBytesUnarchivedInFile, - currentBytesUnarchived, - totalUncompressedBytesInArchive, - totalFilesInArchive, - bytestream.getNumBytesRead(), - )); -}; - - -class TarLocalFile { - // takes a ByteStream and parses out the local file information - constructor(bstream) { - this.isValid = false; - - let bytesRead = 0; - - // Read in the header block - this.name = readCleanString(bstream, 100); - this.mode = readCleanString(bstream, 8); - this.uid = readCleanString(bstream, 8); - this.gid = readCleanString(bstream, 8); - this.size = parseInt(readCleanString(bstream, 12), 8); - this.mtime = readCleanString(bstream, 12); - this.chksum = readCleanString(bstream, 8); - this.typeflag = readCleanString(bstream, 1); - this.linkname = readCleanString(bstream, 100); - this.maybeMagic = readCleanString(bstream, 6); - - if (this.maybeMagic == "ustar") { - this.version = readCleanString(bstream, 2); - this.uname = readCleanString(bstream, 32); - this.gname = readCleanString(bstream, 32); - this.devmajor = readCleanString(bstream, 8); - this.devminor = readCleanString(bstream, 8); - this.prefix = readCleanString(bstream, 155); - - if (this.prefix.length) { - this.name = this.prefix + this.name; - } - bstream.readBytes(12); // 512 - 500in - } else { - bstream.readBytes(255); // 512 - 257 - } - - bytesRead += 512; - - // Done header, now rest of blocks are the file contents. - this.filename = this.name; - this.fileData = null; - - info("Untarring file '" + this.filename + "'"); - info(" size = " + this.size); - info(" typeflag = " + this.typeflag); - - // A regular file. - if (this.typeflag == 0) { - info(" This is a regular file."); - const sizeInBytes = parseInt(this.size); - this.fileData = new Uint8Array(bstream.readBytes(sizeInBytes)); - bytesRead += sizeInBytes; - if (this.name.length > 0 && this.size > 0 && this.fileData && this.fileData.buffer) { - this.isValid = true; - } - - // Round up to 512-byte blocks. - const remaining = 512 - bytesRead % 512; - if (remaining > 0 && remaining < 512) { - bstream.readBytes(remaining); - } - } else if (this.typeflag == 5) { - info(" This is a directory.") - } - } -} - -const untar = function() { - let bstream = bytestream.tee(); - - // While we don't encounter an empty block, keep making TarLocalFiles. - while (bstream.peekNumber(4) != 0) { - const oneLocalFile = new TarLocalFile(bstream); - if (oneLocalFile && oneLocalFile.isValid) { - // If we make it to this point and haven't thrown an error, we have successfully - // read in the data for a local file, so we can update the actual bytestream. - bytestream = bstream.tee(); - - allLocalFiles.push(oneLocalFile); - totalUncompressedBytesInArchive += oneLocalFile.size; - - // update progress - currentFilename = oneLocalFile.filename; - currentFileNumber = totalFilesInArchive++; - currentBytesUnarchivedInFile = oneLocalFile.size; - currentBytesUnarchived += oneLocalFile.size; - postMessage(new bitjs.archive.UnarchiveExtractEvent(oneLocalFile)); - postProgress(); - } - } - totalFilesInArchive = allLocalFiles.length; - - postProgress(); - - bytestream = bstream.tee(); -}; - -// event.data.file has the first ArrayBuffer. -// event.data.bytes has all subsequent ArrayBuffers. -onmessage = function(event) { - const bytes = event.data.file || event.data.bytes; - logToConsole = !!event.data.logToConsole; - - // This is the very first time we have been called. Initialize the bytestream. - if (!bytestream) { - bytestream = new bitjs.io.ByteStream(bytes); - } else { - bytestream.push(bytes); - } - - if (unarchiveState === UnarchiveState.NOT_STARTED) { - currentFilename = ""; - currentFileNumber = 0; - currentBytesUnarchivedInFile = 0; - currentBytesUnarchived = 0; - totalUncompressedBytesInArchive = 0; - totalFilesInArchive = 0; - allLocalFiles = []; - - postMessage(new bitjs.archive.UnarchiveStartEvent()); - - unarchiveState = UnarchiveState.UNARCHIVING; - - postProgress(); - } - - if (unarchiveState === UnarchiveState.UNARCHIVING || - unarchiveState === UnarchiveState.WAITING) { - try { - untar(); - unarchiveState = UnarchiveState.FINISHED; - postMessage(new bitjs.archive.UnarchiveFinishEvent()); - } catch (e) { - if (typeof e === 'string' && e.startsWith('Error! Overflowed')) { - // Overrun the buffer. - unarchiveState = UnarchiveState.WAITING; - } else { - console.error('Found an error while untarring'); - console.dir(e); - throw e; - } - } - } -}; diff --git a/cps/templates/readcbr.html b/cps/templates/readcbr.html index 9fdce0f8..8b2653b3 100644 --- a/cps/templates/readcbr.html +++ b/cps/templates/readcbr.html @@ -1,9 +1,9 @@ + Comic Reader - Comic Reader @@ -15,7 +15,7 @@ - +