diff --git a/build/kuromoji.js b/build/kuromoji.js index f0f4ae9183ff8965fda64a2042f29936f76506d1..fa481f01cb927401e89e432d39ac14950ce28260 100644 --- a/build/kuromoji.js +++ b/build/kuromoji.js @@ -1,5 +1,5 @@ -(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.kuromoji = f()}})(function(){var define,module,exports;return (function(){function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o -1 && value % 1 == 0 && value < length); + (type == 'number' || + (type != 'symbol' && reIsUint.test(value))) && + (value > -1 && value % 1 == 0 && value < length); } /** `Object#toString` result references. */ @@ -755,6 +758,14 @@ var freeProcess = moduleExports$1 && freeGlobal.process; /** Used to access faster Node.js helpers. */ var nodeUtil = (function() { try { + // Use `util.types` for Node.js 10+. + var types = freeModule$1 && freeModule$1.require && freeModule$1.require('util').types; + + if (types) { + return types; + } + + // Legacy `process.binding('util')` for Node.js < 10. return freeProcess && freeProcess.binding && freeProcess.binding('util'); } catch (e) {} }()); @@ -939,6 +950,9 @@ function createObjectIterator(obj) { var len = okeys.length; return function next() { var key = okeys[++i]; + if (key === '__proto__') { + return next(); + } return i < len ? {value: obj[key], key: key} : null; }; } @@ -970,6 +984,7 @@ function _eachOfLimit(limit) { var nextElem = iterator(obj); var done = false; var running = 0; + var looping = false; function iterateeCallback(err, value) { running -= 1; @@ -981,12 +996,13 @@ function _eachOfLimit(limit) { done = true; return callback(null); } - else { + else if (!looping) { replenish(); } } function replenish () { + looping = true; while (running < limit && !done) { var elem = nextElem(); if (elem === null) { @@ -999,6 +1015,7 @@ function _eachOfLimit(limit) { running += 1; iteratee(elem.value, elem.key, onlyOnce(iterateeCallback)); } + looping = false; } replenish(); @@ -3819,7 +3836,7 @@ function memoize(fn, hasher) { /** * Calls `callback` on a later loop around the event loop. In Node.js this just - * calls `process.nextTicl`. In the browser it will use `setImmediate` if + * calls `process.nextTick`. In the browser it will use `setImmediate` if * available, otherwise `setTimeout(callback, 0)`, which means other higher * priority events may precede the execution of `callback`. * @@ -5596,8 +5613,8 @@ Object.defineProperty(exports, '__esModule', { value: true }); }))); -}).call(this,require('_process'),typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) -},{"_process":4}],2:[function(require,module,exports){ +}).call(this)}).call(this,require('_process'),typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {},require("timers").setImmediate) +},{"_process":5,"timers":6}],2:[function(require,module,exports){ // Copyright (c) 2014 Takuya Asano All Rights Reserved. (function () { @@ -6391,328 +6408,2792 @@ Object.defineProperty(exports, '__esModule', { value: true }); })(); },{}],3:[function(require,module,exports){ -(function (process){ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -// resolves . and .. elements in a path array with directory names there -// must be no slashes, empty elements, or device names (c:\) in the array -// (so also no leading and trailing slashes - it does not distinguish -// relative and absolute paths) -function normalizeArray(parts, allowAboveRoot) { - // if the path tries to go above the root, `up` ends up > 0 - var up = 0; - for (var i = parts.length - 1; i >= 0; i--) { - var last = parts[i]; - if (last === '.') { - parts.splice(i, 1); - } else if (last === '..') { - parts.splice(i, 1); - up++; - } else if (up) { - parts.splice(i, 1); - up--; - } - } - - // if the path is allowed to go above the root, restore leading ..s - if (allowAboveRoot) { - for (; up--; up) { - parts.unshift('..'); +"use strict"; +// DEFLATE is a complex format; to read this code, you should probably check the RFC first: +// https://tools.ietf.org/html/rfc1951 +// You may also wish to take a look at the guide I made about this program: +// https://gist.github.com/101arrowz/253f31eb5abc3d9275ab943003ffecad +// Some of the following code is similar to that of UZIP.js: +// https://github.com/photopea/UZIP.js +// However, the vast majority of the codebase has diverged from UZIP.js to increase performance and reduce bundle size. +// Sometimes 0 will appear where -1 would be more appropriate. This is because using a uint +// is better for memory in most engines (I *think*). +var node_worker_1 = require("./node-worker.cjs"); +// aliases for shorter compressed code (most minifers don't do this) +var u8 = Uint8Array, u16 = Uint16Array, i32 = Int32Array; +// fixed length extra bits +var fleb = new u8([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, /* unused */ 0, 0, /* impossible */ 0]); +// fixed distance extra bits +var fdeb = new u8([0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, /* unused */ 0, 0]); +// code length index map +var clim = new u8([16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]); +// get base, reverse index map from extra bits +var freb = function (eb, start) { + var b = new u16(31); + for (var i = 0; i < 31; ++i) { + b[i] = start += 1 << eb[i - 1]; + } + // numbers here are at max 18 bits + var r = new i32(b[30]); + for (var i = 1; i < 30; ++i) { + for (var j = b[i]; j < b[i + 1]; ++j) { + r[j] = ((j - b[i]) << 5) | i; + } } - } - - return parts; -} - -// Split a filename into [root, dir, basename, ext], unix version -// 'root' is just a slash, or nothing. -var splitPathRe = - /^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/; -var splitPath = function(filename) { - return splitPathRe.exec(filename).slice(1); + return { b: b, r: r }; }; - -// path.resolve([from ...], to) -// posix version -exports.resolve = function() { - var resolvedPath = '', - resolvedAbsolute = false; - - for (var i = arguments.length - 1; i >= -1 && !resolvedAbsolute; i--) { - var path = (i >= 0) ? arguments[i] : process.cwd(); - - // Skip empty and invalid entries - if (typeof path !== 'string') { - throw new TypeError('Arguments to path.resolve must be strings'); - } else if (!path) { - continue; +var _a = freb(fleb, 2), fl = _a.b, revfl = _a.r; +// we can ignore the fact that the other numbers are wrong; they never happen anyway +fl[28] = 258, revfl[258] = 28; +var _b = freb(fdeb, 0), fd = _b.b, revfd = _b.r; +// map of value to reverse (assuming 16 bits) +var rev = new u16(32768); +for (var i = 0; i < 32768; ++i) { + // reverse table algorithm from SO + var x = ((i & 0xAAAA) >> 1) | ((i & 0x5555) << 1); + x = ((x & 0xCCCC) >> 2) | ((x & 0x3333) << 2); + x = ((x & 0xF0F0) >> 4) | ((x & 0x0F0F) << 4); + rev[i] = (((x & 0xFF00) >> 8) | ((x & 0x00FF) << 8)) >> 1; +} +// create huffman tree from u8 "map": index -> code length for code index +// mb (max bits) must be at most 15 +// TODO: optimize/split up? +var hMap = (function (cd, mb, r) { + var s = cd.length; + // index + var i = 0; + // u16 "map": index -> # of codes with bit length = index + var l = new u16(mb); + // length of cd must be 288 (total # of codes) + for (; i < s; ++i) { + if (cd[i]) + ++l[cd[i] - 1]; + } + // u16 "map": index -> minimum code for bit length = index + var le = new u16(mb); + for (i = 1; i < mb; ++i) { + le[i] = (le[i - 1] + l[i - 1]) << 1; + } + var co; + if (r) { + // u16 "map": index -> number of actual bits, symbol for code + co = new u16(1 << mb); + // bits to remove for reverser + var rvb = 15 - mb; + for (i = 0; i < s; ++i) { + // ignore 0 lengths + if (cd[i]) { + // num encoding both symbol and bits read + var sv = (i << 4) | cd[i]; + // free bits + var r_1 = mb - cd[i]; + // start value + var v = le[cd[i] - 1]++ << r_1; + // m is end value + for (var m = v | ((1 << r_1) - 1); v <= m; ++v) { + // every 16 bit value starting with the code yields the same result + co[rev[v] >> rvb] = sv; + } + } + } } - - resolvedPath = path + '/' + resolvedPath; - resolvedAbsolute = path.charAt(0) === '/'; - } - - // At this point the path should be resolved to a full absolute path, but - // handle relative paths to be safe (might happen when process.cwd() fails) - - // Normalize the path - resolvedPath = normalizeArray(filter(resolvedPath.split('/'), function(p) { - return !!p; - }), !resolvedAbsolute).join('/'); - - return ((resolvedAbsolute ? '/' : '') + resolvedPath) || '.'; + else { + co = new u16(s); + for (i = 0; i < s; ++i) { + if (cd[i]) { + co[i] = rev[le[cd[i] - 1]++] >> (15 - cd[i]); + } + } + } + return co; +}); +// fixed length tree +var flt = new u8(288); +for (var i = 0; i < 144; ++i) + flt[i] = 8; +for (var i = 144; i < 256; ++i) + flt[i] = 9; +for (var i = 256; i < 280; ++i) + flt[i] = 7; +for (var i = 280; i < 288; ++i) + flt[i] = 8; +// fixed distance tree +var fdt = new u8(32); +for (var i = 0; i < 32; ++i) + fdt[i] = 5; +// fixed length map +var flm = /*#__PURE__*/ hMap(flt, 9, 0), flrm = /*#__PURE__*/ hMap(flt, 9, 1); +// fixed distance map +var fdm = /*#__PURE__*/ hMap(fdt, 5, 0), fdrm = /*#__PURE__*/ hMap(fdt, 5, 1); +// find max of array +var max = function (a) { + var m = a[0]; + for (var i = 1; i < a.length; ++i) { + if (a[i] > m) + m = a[i]; + } + return m; }; - -// path.normalize(path) -// posix version -exports.normalize = function(path) { - var isAbsolute = exports.isAbsolute(path), - trailingSlash = substr(path, -1) === '/'; - - // Normalize the path - path = normalizeArray(filter(path.split('/'), function(p) { - return !!p; - }), !isAbsolute).join('/'); - - if (!path && !isAbsolute) { - path = '.'; - } - if (path && trailingSlash) { - path += '/'; - } - - return (isAbsolute ? '/' : '') + path; +// read d, starting at bit p and mask with m +var bits = function (d, p, m) { + var o = (p / 8) | 0; + return ((d[o] | (d[o + 1] << 8)) >> (p & 7)) & m; }; - -// posix version -exports.isAbsolute = function(path) { - return path.charAt(0) === '/'; +// read d, starting at bit p continuing for at least 16 bits +var bits16 = function (d, p) { + var o = (p / 8) | 0; + return ((d[o] | (d[o + 1] << 8) | (d[o + 2] << 16)) >> (p & 7)); }; - -// posix version -exports.join = function() { - var paths = Array.prototype.slice.call(arguments, 0); - return exports.normalize(filter(paths, function(p, index) { - if (typeof p !== 'string') { - throw new TypeError('Arguments to path.join must be strings'); - } - return p; - }).join('/')); +// get end of byte +var shft = function (p) { return ((p + 7) / 8) | 0; }; +// typed array slice - allows garbage collector to free original reference, +// while being more compatible than .slice +var slc = function (v, s, e) { + if (s == null || s < 0) + s = 0; + if (e == null || e > v.length) + e = v.length; + // can't use .constructor in case user-supplied + return new u8(v.subarray(s, e)); }; - - -// path.relative(from, to) -// posix version -exports.relative = function(from, to) { - from = exports.resolve(from).substr(1); - to = exports.resolve(to).substr(1); - - function trim(arr) { - var start = 0; - for (; start < arr.length; start++) { - if (arr[start] !== '') break; - } - - var end = arr.length - 1; - for (; end >= 0; end--) { - if (arr[end] !== '') break; +/** + * Codes for errors generated within this library + */ +exports.FlateErrorCode = { + UnexpectedEOF: 0, + InvalidBlockType: 1, + InvalidLengthLiteral: 2, + InvalidDistance: 3, + StreamFinished: 4, + NoStreamHandler: 5, + InvalidHeader: 6, + NoCallback: 7, + InvalidUTF8: 8, + ExtraFieldTooLong: 9, + InvalidDate: 10, + FilenameTooLong: 11, + StreamFinishing: 12, + InvalidZipData: 13, + UnknownCompressionMethod: 14 +}; +// error codes +var ec = [ + 'unexpected EOF', + 'invalid block type', + 'invalid length/literal', + 'invalid distance', + 'stream finished', + 'no stream handler', + , + 'no callback', + 'invalid UTF-8 data', + 'extra field too long', + 'date not in range 1980-2099', + 'filename too long', + 'stream finishing', + 'invalid zip data' + // determined by unknown compression method +]; +; +var err = function (ind, msg, nt) { + var e = new Error(msg || ec[ind]); + e.code = ind; + if (Error.captureStackTrace) + Error.captureStackTrace(e, err); + if (!nt) + throw e; + return e; +}; +// expands raw DEFLATE data +var inflt = function (dat, st, buf, dict) { + // source length dict length + var sl = dat.length, dl = dict ? dict.length : 0; + if (!sl || st.f && !st.l) + return buf || new u8(0); + var noBuf = !buf; + // have to estimate size + var resize = noBuf || st.i != 2; + // no state + var noSt = st.i; + // Assumes roughly 33% compression ratio average + if (noBuf) + buf = new u8(sl * 3); + // ensure buffer can fit at least l elements + var cbuf = function (l) { + var bl = buf.length; + // need to increase size to fit + if (l > bl) { + // Double or set to necessary, whichever is greater + var nbuf = new u8(Math.max(bl * 2, l)); + nbuf.set(buf); + buf = nbuf; + } + }; + // last chunk bitpos bytes + var final = st.f || 0, pos = st.p || 0, bt = st.b || 0, lm = st.l, dm = st.d, lbt = st.m, dbt = st.n; + // total bits + var tbts = sl * 8; + do { + if (!lm) { + // BFINAL - this is only 1 when last chunk is next + final = bits(dat, pos, 1); + // type: 0 = no compression, 1 = fixed huffman, 2 = dynamic huffman + var type = bits(dat, pos + 1, 3); + pos += 3; + if (!type) { + // go to end of byte boundary + var s = shft(pos) + 4, l = dat[s - 4] | (dat[s - 3] << 8), t = s + l; + if (t > sl) { + if (noSt) + err(0); + break; + } + // ensure size + if (resize) + cbuf(bt + l); + // Copy over uncompressed data + buf.set(dat.subarray(s, t), bt); + // Get new bitpos, update byte count + st.b = bt += l, st.p = pos = t * 8, st.f = final; + continue; + } + else if (type == 1) + lm = flrm, dm = fdrm, lbt = 9, dbt = 5; + else if (type == 2) { + // literal lengths + var hLit = bits(dat, pos, 31) + 257, hcLen = bits(dat, pos + 10, 15) + 4; + var tl = hLit + bits(dat, pos + 5, 31) + 1; + pos += 14; + // length+distance tree + var ldt = new u8(tl); + // code length tree + var clt = new u8(19); + for (var i = 0; i < hcLen; ++i) { + // use index map to get real code + clt[clim[i]] = bits(dat, pos + i * 3, 7); + } + pos += hcLen * 3; + // code lengths bits + var clb = max(clt), clbmsk = (1 << clb) - 1; + // code lengths map + var clm = hMap(clt, clb, 1); + for (var i = 0; i < tl;) { + var r = clm[bits(dat, pos, clbmsk)]; + // bits read + pos += r & 15; + // symbol + var s = r >> 4; + // code length to copy + if (s < 16) { + ldt[i++] = s; + } + else { + // copy count + var c = 0, n = 0; + if (s == 16) + n = 3 + bits(dat, pos, 3), pos += 2, c = ldt[i - 1]; + else if (s == 17) + n = 3 + bits(dat, pos, 7), pos += 3; + else if (s == 18) + n = 11 + bits(dat, pos, 127), pos += 7; + while (n--) + ldt[i++] = c; + } + } + // length tree distance tree + var lt = ldt.subarray(0, hLit), dt = ldt.subarray(hLit); + // max length bits + lbt = max(lt); + // max dist bits + dbt = max(dt); + lm = hMap(lt, lbt, 1); + dm = hMap(dt, dbt, 1); + } + else + err(1); + if (pos > tbts) { + if (noSt) + err(0); + break; + } + } + // Make sure the buffer can hold this + the largest possible addition + // Maximum chunk size (practically, theoretically infinite) is 2^17 + if (resize) + cbuf(bt + 131072); + var lms = (1 << lbt) - 1, dms = (1 << dbt) - 1; + var lpos = pos; + for (;; lpos = pos) { + // bits read, code + var c = lm[bits16(dat, pos) & lms], sym = c >> 4; + pos += c & 15; + if (pos > tbts) { + if (noSt) + err(0); + break; + } + if (!c) + err(2); + if (sym < 256) + buf[bt++] = sym; + else if (sym == 256) { + lpos = pos, lm = null; + break; + } + else { + var add = sym - 254; + // no extra bits needed if less + if (sym > 264) { + // index + var i = sym - 257, b = fleb[i]; + add = bits(dat, pos, (1 << b) - 1) + fl[i]; + pos += b; + } + // dist + var d = dm[bits16(dat, pos) & dms], dsym = d >> 4; + if (!d) + err(3); + pos += d & 15; + var dt = fd[dsym]; + if (dsym > 3) { + var b = fdeb[dsym]; + dt += bits16(dat, pos) & (1 << b) - 1, pos += b; + } + if (pos > tbts) { + if (noSt) + err(0); + break; + } + if (resize) + cbuf(bt + 131072); + var end = bt + add; + if (bt < dt) { + var shift = dl - dt, dend = Math.min(dt, end); + if (shift + bt < 0) + err(3); + for (; bt < dend; ++bt) + buf[bt] = dict[shift + bt]; + } + for (; bt < end; ++bt) + buf[bt] = buf[bt - dt]; + } + } + st.l = lm, st.p = lpos, st.b = bt, st.f = final; + if (lm) + final = 1, st.m = lbt, st.d = dm, st.n = dbt; + } while (!final); + // don't reallocate for streams or user buffers + return bt != buf.length && noBuf ? slc(buf, 0, bt) : buf.subarray(0, bt); +}; +// starting at p, write the minimum number of bits that can hold v to d +var wbits = function (d, p, v) { + v <<= p & 7; + var o = (p / 8) | 0; + d[o] |= v; + d[o + 1] |= v >> 8; +}; +// starting at p, write the minimum number of bits (>8) that can hold v to d +var wbits16 = function (d, p, v) { + v <<= p & 7; + var o = (p / 8) | 0; + d[o] |= v; + d[o + 1] |= v >> 8; + d[o + 2] |= v >> 16; +}; +// creates code lengths from a frequency table +var hTree = function (d, mb) { + // Need extra info to make a tree + var t = []; + for (var i = 0; i < d.length; ++i) { + if (d[i]) + t.push({ s: i, f: d[i] }); + } + var s = t.length; + var t2 = t.slice(); + if (!s) + return { t: et, l: 0 }; + if (s == 1) { + var v = new u8(t[0].s + 1); + v[t[0].s] = 1; + return { t: v, l: 1 }; + } + t.sort(function (a, b) { return a.f - b.f; }); + // after i2 reaches last ind, will be stopped + // freq must be greater than largest possible number of symbols + t.push({ s: -1, f: 25001 }); + var l = t[0], r = t[1], i0 = 0, i1 = 1, i2 = 2; + t[0] = { s: -1, f: l.f + r.f, l: l, r: r }; + // efficient algorithm from UZIP.js + // i0 is lookbehind, i2 is lookahead - after processing two low-freq + // symbols that combined have high freq, will start processing i2 (high-freq, + // non-composite) symbols instead + // see https://reddit.com/r/photopea/comments/ikekht/uzipjs_questions/ + while (i1 != s - 1) { + l = t[t[i0].f < t[i2].f ? i0++ : i2++]; + r = t[i0 != i1 && t[i0].f < t[i2].f ? i0++ : i2++]; + t[i1++] = { s: -1, f: l.f + r.f, l: l, r: r }; + } + var maxSym = t2[0].s; + for (var i = 1; i < s; ++i) { + if (t2[i].s > maxSym) + maxSym = t2[i].s; + } + // code lengths + var tr = new u16(maxSym + 1); + // max bits in tree + var mbt = ln(t[i1 - 1], tr, 0); + if (mbt > mb) { + // more algorithms from UZIP.js + // TODO: find out how this code works (debt) + // ind debt + var i = 0, dt = 0; + // left cost + var lft = mbt - mb, cst = 1 << lft; + t2.sort(function (a, b) { return tr[b.s] - tr[a.s] || a.f - b.f; }); + for (; i < s; ++i) { + var i2_1 = t2[i].s; + if (tr[i2_1] > mb) { + dt += cst - (1 << (mbt - tr[i2_1])); + tr[i2_1] = mb; + } + else + break; + } + dt >>= lft; + while (dt > 0) { + var i2_2 = t2[i].s; + if (tr[i2_2] < mb) + dt -= 1 << (mb - tr[i2_2]++ - 1); + else + ++i; + } + for (; i >= 0 && dt; --i) { + var i2_3 = t2[i].s; + if (tr[i2_3] == mb) { + --tr[i2_3]; + ++dt; + } + } + mbt = mb; } - - if (start > end) return []; - return arr.slice(start, end - start + 1); - } - - var fromParts = trim(from.split('/')); - var toParts = trim(to.split('/')); - - var length = Math.min(fromParts.length, toParts.length); - var samePartsLength = length; - for (var i = 0; i < length; i++) { - if (fromParts[i] !== toParts[i]) { - samePartsLength = i; - break; + return { t: new u8(tr), l: mbt }; +}; +// get the max length and assign length codes +var ln = function (n, l, d) { + return n.s == -1 + ? Math.max(ln(n.l, l, d + 1), ln(n.r, l, d + 1)) + : (l[n.s] = d); +}; +// length codes generation +var lc = function (c) { + var s = c.length; + // Note that the semicolon was intentional + while (s && !c[--s]) + ; + var cl = new u16(++s); + // ind num streak + var cli = 0, cln = c[0], cls = 1; + var w = function (v) { cl[cli++] = v; }; + for (var i = 1; i <= s; ++i) { + if (c[i] == cln && i != s) + ++cls; + else { + if (!cln && cls > 2) { + for (; cls > 138; cls -= 138) + w(32754); + if (cls > 2) { + w(cls > 10 ? ((cls - 11) << 5) | 28690 : ((cls - 3) << 5) | 12305); + cls = 0; + } + } + else if (cls > 3) { + w(cln), --cls; + for (; cls > 6; cls -= 6) + w(8304); + if (cls > 2) + w(((cls - 3) << 5) | 8208), cls = 0; + } + while (cls--) + w(cln); + cls = 1; + cln = c[i]; + } } - } - - var outputParts = []; - for (var i = samePartsLength; i < fromParts.length; i++) { - outputParts.push('..'); - } - - outputParts = outputParts.concat(toParts.slice(samePartsLength)); - - return outputParts.join('/'); + return { c: cl.subarray(0, cli), n: s }; }; - -exports.sep = '/'; -exports.delimiter = ':'; - -exports.dirname = function(path) { - var result = splitPath(path), - root = result[0], - dir = result[1]; - - if (!root && !dir) { - // No dirname whatsoever - return '.'; - } - - if (dir) { - // It has a dirname, strip trailing slash - dir = dir.substr(0, dir.length - 1); - } - - return root + dir; +// calculate the length of output from tree, code lengths +var clen = function (cf, cl) { + var l = 0; + for (var i = 0; i < cl.length; ++i) + l += cf[i] * cl[i]; + return l; }; - - -exports.basename = function(path, ext) { - var f = splitPath(path)[2]; - // TODO: make this comparison case-insensitive on windows? - if (ext && f.substr(-1 * ext.length) === ext) { - f = f.substr(0, f.length - ext.length); - } - return f; +// writes a fixed block +// returns the new bit pos +var wfblk = function (out, pos, dat) { + // no need to write 00 as type: TypedArray defaults to 0 + var s = dat.length; + var o = shft(pos + 2); + out[o] = s & 255; + out[o + 1] = s >> 8; + out[o + 2] = out[o] ^ 255; + out[o + 3] = out[o + 1] ^ 255; + for (var i = 0; i < s; ++i) + out[o + i + 4] = dat[i]; + return (o + 4 + s) * 8; }; - - -exports.extname = function(path) { - return splitPath(path)[3]; +// writes a block +var wblk = function (dat, out, final, syms, lf, df, eb, li, bs, bl, p) { + wbits(out, p++, final); + ++lf[256]; + var _a = hTree(lf, 15), dlt = _a.t, mlb = _a.l; + var _b = hTree(df, 15), ddt = _b.t, mdb = _b.l; + var _c = lc(dlt), lclt = _c.c, nlc = _c.n; + var _d = lc(ddt), lcdt = _d.c, ndc = _d.n; + var lcfreq = new u16(19); + for (var i = 0; i < lclt.length; ++i) + ++lcfreq[lclt[i] & 31]; + for (var i = 0; i < lcdt.length; ++i) + ++lcfreq[lcdt[i] & 31]; + var _e = hTree(lcfreq, 7), lct = _e.t, mlcb = _e.l; + var nlcc = 19; + for (; nlcc > 4 && !lct[clim[nlcc - 1]]; --nlcc) + ; + var flen = (bl + 5) << 3; + var ftlen = clen(lf, flt) + clen(df, fdt) + eb; + var dtlen = clen(lf, dlt) + clen(df, ddt) + eb + 14 + 3 * nlcc + clen(lcfreq, lct) + 2 * lcfreq[16] + 3 * lcfreq[17] + 7 * lcfreq[18]; + if (bs >= 0 && flen <= ftlen && flen <= dtlen) + return wfblk(out, p, dat.subarray(bs, bs + bl)); + var lm, ll, dm, dl; + wbits(out, p, 1 + (dtlen < ftlen)), p += 2; + if (dtlen < ftlen) { + lm = hMap(dlt, mlb, 0), ll = dlt, dm = hMap(ddt, mdb, 0), dl = ddt; + var llm = hMap(lct, mlcb, 0); + wbits(out, p, nlc - 257); + wbits(out, p + 5, ndc - 1); + wbits(out, p + 10, nlcc - 4); + p += 14; + for (var i = 0; i < nlcc; ++i) + wbits(out, p + 3 * i, lct[clim[i]]); + p += 3 * nlcc; + var lcts = [lclt, lcdt]; + for (var it = 0; it < 2; ++it) { + var clct = lcts[it]; + for (var i = 0; i < clct.length; ++i) { + var len = clct[i] & 31; + wbits(out, p, llm[len]), p += lct[len]; + if (len > 15) + wbits(out, p, (clct[i] >> 5) & 127), p += clct[i] >> 12; + } + } + } + else { + lm = flm, ll = flt, dm = fdm, dl = fdt; + } + for (var i = 0; i < li; ++i) { + var sym = syms[i]; + if (sym > 255) { + var len = (sym >> 18) & 31; + wbits16(out, p, lm[len + 257]), p += ll[len + 257]; + if (len > 7) + wbits(out, p, (sym >> 23) & 31), p += fleb[len]; + var dst = sym & 31; + wbits16(out, p, dm[dst]), p += dl[dst]; + if (dst > 3) + wbits16(out, p, (sym >> 5) & 8191), p += fdeb[dst]; + } + else { + wbits16(out, p, lm[sym]), p += ll[sym]; + } + } + wbits16(out, p, lm[256]); + return p + ll[256]; }; - -function filter (xs, f) { - if (xs.filter) return xs.filter(f); - var res = []; - for (var i = 0; i < xs.length; i++) { - if (f(xs[i], i, xs)) res.push(xs[i]); +// deflate options (nice << 13) | chain +var deo = /*#__PURE__*/ new i32([65540, 131080, 131088, 131104, 262176, 1048704, 1048832, 2114560, 2117632]); +// empty +var et = /*#__PURE__*/ new u8(0); +// compresses data into a raw DEFLATE buffer +var dflt = function (dat, lvl, plvl, pre, post, st) { + var s = st.z || dat.length; + var o = new u8(pre + s + 5 * (1 + Math.ceil(s / 7000)) + post); + // writing to this writes to the output buffer + var w = o.subarray(pre, o.length - post); + var lst = st.l; + var pos = (st.r || 0) & 7; + if (lvl) { + if (pos) + w[0] = st.r >> 3; + var opt = deo[lvl - 1]; + var n = opt >> 13, c = opt & 8191; + var msk_1 = (1 << plvl) - 1; + // prev 2-byte val map curr 2-byte val map + var prev = st.p || new u16(32768), head = st.h || new u16(msk_1 + 1); + var bs1_1 = Math.ceil(plvl / 3), bs2_1 = 2 * bs1_1; + var hsh = function (i) { return (dat[i] ^ (dat[i + 1] << bs1_1) ^ (dat[i + 2] << bs2_1)) & msk_1; }; + // 24576 is an arbitrary number of maximum symbols per block + // 424 buffer for last block + var syms = new i32(25000); + // length/literal freq distance freq + var lf = new u16(288), df = new u16(32); + // l/lcnt exbits index l/lind waitdx blkpos + var lc_1 = 0, eb = 0, i = st.i || 0, li = 0, wi = st.w || 0, bs = 0; + for (; i + 2 < s; ++i) { + // hash value + var hv = hsh(i); + // index mod 32768 previous index mod + var imod = i & 32767, pimod = head[hv]; + prev[imod] = pimod; + head[hv] = imod; + // We always should modify head and prev, but only add symbols if + // this data is not yet processed ("wait" for wait index) + if (wi <= i) { + // bytes remaining + var rem = s - i; + if ((lc_1 > 7000 || li > 24576) && (rem > 423 || !lst)) { + pos = wblk(dat, w, 0, syms, lf, df, eb, li, bs, i - bs, pos); + li = lc_1 = eb = 0, bs = i; + for (var j = 0; j < 286; ++j) + lf[j] = 0; + for (var j = 0; j < 30; ++j) + df[j] = 0; + } + // len dist chain + var l = 2, d = 0, ch_1 = c, dif = imod - pimod & 32767; + if (rem > 2 && hv == hsh(i - dif)) { + var maxn = Math.min(n, rem) - 1; + var maxd = Math.min(32767, i); + // max possible length + // not capped at dif because decompressors implement "rolling" index population + var ml = Math.min(258, rem); + while (dif <= maxd && --ch_1 && imod != pimod) { + if (dat[i + l] == dat[i + l - dif]) { + var nl = 0; + for (; nl < ml && dat[i + nl] == dat[i + nl - dif]; ++nl) + ; + if (nl > l) { + l = nl, d = dif; + // break out early when we reach "nice" (we are satisfied enough) + if (nl > maxn) + break; + // now, find the rarest 2-byte sequence within this + // length of literals and search for that instead. + // Much faster than just using the start + var mmd = Math.min(dif, nl - 2); + var md = 0; + for (var j = 0; j < mmd; ++j) { + var ti = i - dif + j & 32767; + var pti = prev[ti]; + var cd = ti - pti & 32767; + if (cd > md) + md = cd, pimod = ti; + } + } + } + // check the previous match + imod = pimod, pimod = prev[imod]; + dif += imod - pimod & 32767; + } + } + // d will be nonzero only when a match was found + if (d) { + // store both dist and len data in one int32 + // Make sure this is recognized as a len/dist with 28th bit (2^28) + syms[li++] = 268435456 | (revfl[l] << 18) | revfd[d]; + var lin = revfl[l] & 31, din = revfd[d] & 31; + eb += fleb[lin] + fdeb[din]; + ++lf[257 + lin]; + ++df[din]; + wi = i + l; + ++lc_1; + } + else { + syms[li++] = dat[i]; + ++lf[dat[i]]; + } + } + } + for (i = Math.max(i, wi); i < s; ++i) { + syms[li++] = dat[i]; + ++lf[dat[i]]; + } + pos = wblk(dat, w, lst, syms, lf, df, eb, li, bs, i - bs, pos); + if (!lst) { + st.r = (pos & 7) | w[(pos / 8) | 0] << 3; + // shft(pos) now 1 less if pos & 7 != 0 + pos -= 7; + st.h = head, st.p = prev, st.i = i, st.w = wi; + } } - return res; -} - -// String.prototype.substr - negative index don't work in IE8 -var substr = 'ab'.substr(-1) === 'b' - ? function (str, start, len) { return str.substr(start, len) } - : function (str, start, len) { - if (start < 0) start = str.length + start; - return str.substr(start, len); + else { + for (var i = st.w || 0; i < s + lst; i += 65535) { + // end + var e = i + 65535; + if (e >= s) { + // write final block + w[(pos / 8) | 0] = lst; + e = s; + } + pos = wfblk(w, pos + 1, dat.subarray(i, e)); + } + st.i = s; } + return slc(o, 0, pre + shft(pos) + post); +}; +// CRC32 table +var crct = /*#__PURE__*/ (function () { + var t = new Int32Array(256); + for (var i = 0; i < 256; ++i) { + var c = i, k = 9; + while (--k) + c = ((c & 1) && -306674912) ^ (c >>> 1); + t[i] = c; + } + return t; +})(); +// CRC32 +var crc = function () { + var c = -1; + return { + p: function (d) { + // closures have awful performance + var cr = c; + for (var i = 0; i < d.length; ++i) + cr = crct[(cr & 255) ^ d[i]] ^ (cr >>> 8); + c = cr; + }, + d: function () { return ~c; } + }; +}; +// Adler32 +var adler = function () { + var a = 1, b = 0; + return { + p: function (d) { + // closures have awful performance + var n = a, m = b; + var l = d.length | 0; + for (var i = 0; i != l;) { + var e = Math.min(i + 2655, l); + for (; i < e; ++i) + m += n += d[i]; + n = (n & 65535) + 15 * (n >> 16), m = (m & 65535) + 15 * (m >> 16); + } + a = n, b = m; + }, + d: function () { + a %= 65521, b %= 65521; + return (a & 255) << 24 | (a & 0xFF00) << 8 | (b & 255) << 8 | (b >> 8); + } + }; +}; ; - -}).call(this,require('_process')) -},{"_process":4}],4:[function(require,module,exports){ -// shim for using process in browser -var process = module.exports = {}; - -// cached from whatever global is present so that test runners that stub it -// don't break things. But we need to wrap it in a try catch in case it is -// wrapped in strict mode code which doesn't define any globals. It's inside a -// function because try/catches deoptimize in certain engines. - -var cachedSetTimeout; -var cachedClearTimeout; - -function defaultSetTimout() { - throw new Error('setTimeout has not been defined'); -} -function defaultClearTimeout () { - throw new Error('clearTimeout has not been defined'); -} -(function () { - try { - if (typeof setTimeout === 'function') { - cachedSetTimeout = setTimeout; - } else { - cachedSetTimeout = defaultSetTimout; +// deflate with opts +var dopt = function (dat, opt, pre, post, st) { + if (!st) { + st = { l: 1 }; + if (opt.dictionary) { + var dict = opt.dictionary.subarray(-32768); + var newDat = new u8(dict.length + dat.length); + newDat.set(dict); + newDat.set(dat, dict.length); + dat = newDat; + st.w = dict.length; } - } catch (e) { - cachedSetTimeout = defaultSetTimout; } - try { - if (typeof clearTimeout === 'function') { - cachedClearTimeout = clearTimeout; - } else { - cachedClearTimeout = defaultClearTimeout; + return dflt(dat, opt.level == null ? 6 : opt.level, opt.mem == null ? (st.l ? Math.ceil(Math.max(8, Math.min(13, Math.log(dat.length))) * 1.5) : 20) : (12 + opt.mem), pre, post, st); +}; +// Walmart object spread +var mrg = function (a, b) { + var o = {}; + for (var k in a) + o[k] = a[k]; + for (var k in b) + o[k] = b[k]; + return o; +}; +// worker clone +// This is possibly the craziest part of the entire codebase, despite how simple it may seem. +// The only parameter to this function is a closure that returns an array of variables outside of the function scope. +// We're going to try to figure out the variable names used in the closure as strings because that is crucial for workerization. +// We will return an object mapping of true variable name to value (basically, the current scope as a JS object). +// The reason we can't just use the original variable names is minifiers mangling the toplevel scope. +// This took me three weeks to figure out how to do. +var wcln = function (fn, fnStr, td) { + var dt = fn(); + var st = fn.toString(); + var ks = st.slice(st.indexOf('[') + 1, st.lastIndexOf(']')).replace(/\s+/g, '').split(','); + for (var i = 0; i < dt.length; ++i) { + var v = dt[i], k = ks[i]; + if (typeof v == 'function') { + fnStr += ';' + k + '='; + var st_1 = v.toString(); + if (v.prototype) { + // for global objects + if (st_1.indexOf('[native code]') != -1) { + var spInd = st_1.indexOf(' ', 8) + 1; + fnStr += st_1.slice(spInd, st_1.indexOf('(', spInd)); + } + else { + fnStr += st_1; + for (var t in v.prototype) + fnStr += ';' + k + '.prototype.' + t + '=' + v.prototype[t].toString(); + } + } + else + fnStr += st_1; } - } catch (e) { - cachedClearTimeout = defaultClearTimeout; - } -} ()) -function runTimeout(fun) { - if (cachedSetTimeout === setTimeout) { - //normal enviroments in sane situations - return setTimeout(fun, 0); + else + td[k] = v; } - // if setTimeout wasn't available but was latter defined - if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { - cachedSetTimeout = setTimeout; - return setTimeout(fun, 0); + return fnStr; +}; +var ch = []; +// clone bufs +var cbfs = function (v) { + var tl = []; + for (var k in v) { + if (v[k].buffer) { + tl.push((v[k] = new v[k].constructor(v[k])).buffer); + } } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedSetTimeout(fun, 0); - } catch(e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedSetTimeout.call(null, fun, 0); - } catch(e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error - return cachedSetTimeout.call(this, fun, 0); + return tl; +}; +// use a worker to execute code +var wrkr = function (fns, init, id, cb) { + if (!ch[id]) { + var fnStr = '', td_1 = {}, m = fns.length - 1; + for (var i = 0; i < m; ++i) + fnStr = wcln(fns[i], fnStr, td_1); + ch[id] = { c: wcln(fns[m], fnStr, td_1), e: td_1 }; + } + var td = mrg({}, ch[id].e); + return (0, node_worker_1.default)(ch[id].c + ';onmessage=function(e){for(var k in e.data)self[k]=e.data[k];onmessage=' + init.toString() + '}', id, td, cbfs(td), cb); +}; +// base async inflate fn +var bInflt = function () { return [u8, u16, i32, fleb, fdeb, clim, fl, fd, flrm, fdrm, rev, ec, hMap, max, bits, bits16, shft, slc, err, inflt, inflateSync, pbf, gopt]; }; +var bDflt = function () { return [u8, u16, i32, fleb, fdeb, clim, revfl, revfd, flm, flt, fdm, fdt, rev, deo, et, hMap, wbits, wbits16, hTree, ln, lc, clen, wfblk, wblk, shft, slc, dflt, dopt, deflateSync, pbf]; }; +// gzip extra +var gze = function () { return [gzh, gzhl, wbytes, crc, crct]; }; +// gunzip extra +var guze = function () { return [gzs, gzl]; }; +// zlib extra +var zle = function () { return [zlh, wbytes, adler]; }; +// unzlib extra +var zule = function () { return [zls]; }; +// post buf +var pbf = function (msg) { return postMessage(msg, [msg.buffer]); }; +// get opts +var gopt = function (o) { return o && { + out: o.size && new u8(o.size), + dictionary: o.dictionary +}; }; +// async helper +var cbify = function (dat, opts, fns, init, id, cb) { + var w = wrkr(fns, init, id, function (err, dat) { + w.terminate(); + cb(err, dat); + }); + w.postMessage([dat, opts], opts.consume ? [dat.buffer] : []); + return function () { w.terminate(); }; +}; +// auto stream +var astrm = function (strm) { + strm.ondata = function (dat, final) { return postMessage([dat, final], [dat.buffer]); }; + return function (ev) { + if (ev.data.length) { + strm.push(ev.data[0], ev.data[1]); + postMessage([ev.data[0].length]); + } + else + strm.flush(); + }; +}; +// async stream attach +var astrmify = function (fns, strm, opts, init, id, flush, ext) { + var t; + var w = wrkr(fns, init, id, function (err, dat) { + if (err) + w.terminate(), strm.ondata.call(strm, err); + else if (!Array.isArray(dat)) + ext(dat); + else if (dat.length == 1) { + strm.queuedSize -= dat[0]; + if (strm.ondrain) + strm.ondrain(dat[0]); + } + else { + if (dat[1]) + w.terminate(); + strm.ondata.call(strm, err, dat[0], dat[1]); } + }); + w.postMessage(opts); + strm.queuedSize = 0; + strm.push = function (d, f) { + if (!strm.ondata) + err(5); + if (t) + strm.ondata(err(4, 0, 1), null, !!f); + strm.queuedSize += d.length; + w.postMessage([d, t = f], [d.buffer]); + }; + strm.terminate = function () { w.terminate(); }; + if (flush) { + strm.flush = function () { w.postMessage([]); }; } - - -} -function runClearTimeout(marker) { - if (cachedClearTimeout === clearTimeout) { - //normal enviroments in sane situations - return clearTimeout(marker); +}; +// read 2 bytes +var b2 = function (d, b) { return d[b] | (d[b + 1] << 8); }; +// read 4 bytes +var b4 = function (d, b) { return (d[b] | (d[b + 1] << 8) | (d[b + 2] << 16) | (d[b + 3] << 24)) >>> 0; }; +var b8 = function (d, b) { return b4(d, b) + (b4(d, b + 4) * 4294967296); }; +// write bytes +var wbytes = function (d, b, v) { + for (; v; ++b) + d[b] = v, v >>>= 8; +}; +// gzip header +var gzh = function (c, o) { + var fn = o.filename; + c[0] = 31, c[1] = 139, c[2] = 8, c[8] = o.level < 2 ? 4 : o.level == 9 ? 2 : 0, c[9] = 3; // assume Unix + if (o.mtime != 0) + wbytes(c, 4, Math.floor(new Date(o.mtime || Date.now()) / 1000)); + if (fn) { + c[3] = 8; + for (var i = 0; i <= fn.length; ++i) + c[i + 10] = fn.charCodeAt(i); } - // if clearTimeout wasn't available but was latter defined - if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { - cachedClearTimeout = clearTimeout; - return clearTimeout(marker); +}; +// gzip footer: -8 to -4 = CRC, -4 to -0 is length +// gzip start +var gzs = function (d) { + if (d[0] != 31 || d[1] != 139 || d[2] != 8) + err(6, 'invalid gzip data'); + var flg = d[3]; + var st = 10; + if (flg & 4) + st += (d[10] | d[11] << 8) + 2; + for (var zs = (flg >> 3 & 1) + (flg >> 4 & 1); zs > 0; zs -= !d[st++]) + ; + return st + (flg & 2); +}; +// gzip length +var gzl = function (d) { + var l = d.length; + return (d[l - 4] | d[l - 3] << 8 | d[l - 2] << 16 | d[l - 1] << 24) >>> 0; +}; +// gzip header length +var gzhl = function (o) { return 10 + (o.filename ? o.filename.length + 1 : 0); }; +// zlib header +var zlh = function (c, o) { + var lv = o.level, fl = lv == 0 ? 0 : lv < 6 ? 1 : lv == 9 ? 3 : 2; + c[0] = 120, c[1] = (fl << 6) | (o.dictionary && 32); + c[1] |= 31 - ((c[0] << 8) | c[1]) % 31; + if (o.dictionary) { + var h = adler(); + h.p(o.dictionary); + wbytes(c, 2, h.d()); } - try { - // when when somebody has screwed with setTimeout but no I.E. maddness - return cachedClearTimeout(marker); - } catch (e){ - try { - // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally - return cachedClearTimeout.call(null, marker); - } catch (e){ - // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. - // Some versions of I.E. have different rules for clearTimeout vs setTimeout - return cachedClearTimeout.call(this, marker); +}; +// zlib start +var zls = function (d, dict) { + if ((d[0] & 15) != 8 || (d[0] >> 4) > 7 || ((d[0] << 8 | d[1]) % 31)) + err(6, 'invalid zlib data'); + if ((d[1] >> 5 & 1) == +!dict) + err(6, 'invalid zlib data: ' + (d[1] & 32 ? 'need' : 'unexpected') + ' dictionary'); + return (d[1] >> 3 & 4) + 2; +}; +function StrmOpt(opts, cb) { + if (typeof opts == 'function') + cb = opts, opts = {}; + this.ondata = cb; + return opts; +} +/** + * Streaming DEFLATE compression + */ +var Deflate = /*#__PURE__*/ (function () { + function Deflate(opts, cb) { + if (typeof opts == 'function') + cb = opts, opts = {}; + this.ondata = cb; + this.o = opts || {}; + this.s = { l: 0, i: 32768, w: 32768, z: 32768 }; + // Buffer length must always be 0 mod 32768 for index calculations to be correct when modifying head and prev + // 98304 = 32768 (lookback) + 65536 (common chunk size) + this.b = new u8(98304); + if (this.o.dictionary) { + var dict = this.o.dictionary.subarray(-32768); + this.b.set(dict, 32768 - dict.length); + this.s.i = 32768 - dict.length; } } + Deflate.prototype.p = function (c, f) { + this.ondata(dopt(c, this.o, 0, 0, this.s), f); + }; + /** + * Pushes a chunk to be deflated + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + Deflate.prototype.push = function (chunk, final) { + if (!this.ondata) + err(5); + if (this.s.l) + err(4); + var endLen = chunk.length + this.s.z; + if (endLen > this.b.length) { + if (endLen > 2 * this.b.length - 32768) { + var newBuf = new u8(endLen & -32768); + newBuf.set(this.b.subarray(0, this.s.z)); + this.b = newBuf; + } + var split = this.b.length - this.s.z; + this.b.set(chunk.subarray(0, split), this.s.z); + this.s.z = this.b.length; + this.p(this.b, false); + this.b.set(this.b.subarray(-32768)); + this.b.set(chunk.subarray(split), 32768); + this.s.z = chunk.length - split + 32768; + this.s.i = 32766, this.s.w = 32768; + } + else { + this.b.set(chunk, this.s.z); + this.s.z += chunk.length; + } + this.s.l = final & 1; + if (this.s.z > this.s.w + 8191 || final) { + this.p(this.b, final || false); + this.s.w = this.s.i, this.s.i -= 2; + } + }; + /** + * Flushes buffered uncompressed data. Useful to immediately retrieve the + * deflated output for small inputs. + */ + Deflate.prototype.flush = function () { + if (!this.ondata) + err(5); + if (this.s.l) + err(4); + this.p(this.b, false); + this.s.w = this.s.i, this.s.i -= 2; + }; + return Deflate; +}()); +exports.Deflate = Deflate; +/** + * Asynchronous streaming DEFLATE compression + */ +var AsyncDeflate = /*#__PURE__*/ (function () { + function AsyncDeflate(opts, cb) { + astrmify([ + bDflt, + function () { return [astrm, Deflate]; } + ], this, StrmOpt.call(this, opts, cb), function (ev) { + var strm = new Deflate(ev.data); + onmessage = astrm(strm); + }, 6, 1); + } + return AsyncDeflate; +}()); +exports.AsyncDeflate = AsyncDeflate; +function deflate(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + return cbify(data, opts, [ + bDflt, + ], function (ev) { return pbf(deflateSync(ev.data[0], ev.data[1])); }, 0, cb); +} +exports.deflate = deflate; +/** + * Compresses data with DEFLATE without any wrapper + * @param data The data to compress + * @param opts The compression options + * @returns The deflated version of the data + */ +function deflateSync(data, opts) { + return dopt(data, opts || {}, 0, 0); +} +exports.deflateSync = deflateSync; +/** + * Streaming DEFLATE decompression + */ +var Inflate = /*#__PURE__*/ (function () { + function Inflate(opts, cb) { + // no StrmOpt here to avoid adding to workerizer + if (typeof opts == 'function') + cb = opts, opts = {}; + this.ondata = cb; + var dict = opts && opts.dictionary && opts.dictionary.subarray(-32768); + this.s = { i: 0, b: dict ? dict.length : 0 }; + this.o = new u8(32768); + this.p = new u8(0); + if (dict) + this.o.set(dict); + } + Inflate.prototype.e = function (c) { + if (!this.ondata) + err(5); + if (this.d) + err(4); + if (!this.p.length) + this.p = c; + else if (c.length) { + var n = new u8(this.p.length + c.length); + n.set(this.p), n.set(c, this.p.length), this.p = n; + } + }; + Inflate.prototype.c = function (final) { + this.s.i = +(this.d = final || false); + var bts = this.s.b; + var dt = inflt(this.p, this.s, this.o); + this.ondata(slc(dt, bts, this.s.b), this.d); + this.o = slc(dt, this.s.b - 32768), this.s.b = this.o.length; + this.p = slc(this.p, (this.s.p / 8) | 0), this.s.p &= 7; + }; + /** + * Pushes a chunk to be inflated + * @param chunk The chunk to push + * @param final Whether this is the final chunk + */ + Inflate.prototype.push = function (chunk, final) { + this.e(chunk), this.c(final); + }; + return Inflate; +}()); +exports.Inflate = Inflate; +/** + * Asynchronous streaming DEFLATE decompression + */ +var AsyncInflate = /*#__PURE__*/ (function () { + function AsyncInflate(opts, cb) { + astrmify([ + bInflt, + function () { return [astrm, Inflate]; } + ], this, StrmOpt.call(this, opts, cb), function (ev) { + var strm = new Inflate(ev.data); + onmessage = astrm(strm); + }, 7, 0); + } + return AsyncInflate; +}()); +exports.AsyncInflate = AsyncInflate; +function inflate(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + return cbify(data, opts, [ + bInflt + ], function (ev) { return pbf(inflateSync(ev.data[0], gopt(ev.data[1]))); }, 1, cb); +} +exports.inflate = inflate; +/** + * Expands DEFLATE data with no wrapper + * @param data The data to decompress + * @param opts The decompression options + * @returns The decompressed version of the data + */ +function inflateSync(data, opts) { + return inflt(data, { i: 2 }, opts && opts.out, opts && opts.dictionary); +} +exports.inflateSync = inflateSync; +// before you yell at me for not just using extends, my reason is that TS inheritance is hard to workerize. +/** + * Streaming GZIP compression + */ +var Gzip = /*#__PURE__*/ (function () { + function Gzip(opts, cb) { + this.c = crc(); + this.l = 0; + this.v = 1; + Deflate.call(this, opts, cb); + } + /** + * Pushes a chunk to be GZIPped + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + Gzip.prototype.push = function (chunk, final) { + this.c.p(chunk); + this.l += chunk.length; + Deflate.prototype.push.call(this, chunk, final); + }; + Gzip.prototype.p = function (c, f) { + var raw = dopt(c, this.o, this.v && gzhl(this.o), f && 8, this.s); + if (this.v) + gzh(raw, this.o), this.v = 0; + if (f) + wbytes(raw, raw.length - 8, this.c.d()), wbytes(raw, raw.length - 4, this.l); + this.ondata(raw, f); + }; + /** + * Flushes buffered uncompressed data. Useful to immediately retrieve the + * GZIPped output for small inputs. + */ + Gzip.prototype.flush = function () { + Deflate.prototype.flush.call(this); + }; + return Gzip; +}()); +exports.Gzip = Gzip; +exports.Compress = Gzip; +/** + * Asynchronous streaming GZIP compression + */ +var AsyncGzip = /*#__PURE__*/ (function () { + function AsyncGzip(opts, cb) { + astrmify([ + bDflt, + gze, + function () { return [astrm, Deflate, Gzip]; } + ], this, StrmOpt.call(this, opts, cb), function (ev) { + var strm = new Gzip(ev.data); + onmessage = astrm(strm); + }, 8, 1); + } + return AsyncGzip; +}()); +exports.AsyncGzip = AsyncGzip; +exports.AsyncCompress = AsyncGzip; +function gzip(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + return cbify(data, opts, [ + bDflt, + gze, + function () { return [gzipSync]; } + ], function (ev) { return pbf(gzipSync(ev.data[0], ev.data[1])); }, 2, cb); +} +exports.gzip = gzip; +exports.compress = gzip; +/** + * Compresses data with GZIP + * @param data The data to compress + * @param opts The compression options + * @returns The gzipped version of the data + */ +function gzipSync(data, opts) { + if (!opts) + opts = {}; + var c = crc(), l = data.length; + c.p(data); + var d = dopt(data, opts, gzhl(opts), 8), s = d.length; + return gzh(d, opts), wbytes(d, s - 8, c.d()), wbytes(d, s - 4, l), d; +} +exports.gzipSync = gzipSync; +exports.compressSync = gzipSync; +/** + * Streaming single or multi-member GZIP decompression + */ +var Gunzip = /*#__PURE__*/ (function () { + function Gunzip(opts, cb) { + this.v = 1; + this.r = 0; + Inflate.call(this, opts, cb); + } + /** + * Pushes a chunk to be GUNZIPped + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + Gunzip.prototype.push = function (chunk, final) { + Inflate.prototype.e.call(this, chunk); + this.r += chunk.length; + if (this.v) { + var p = this.p.subarray(this.v - 1); + var s = p.length > 3 ? gzs(p) : 4; + if (s > p.length) { + if (!final) + return; + } + else if (this.v > 1 && this.onmember) { + this.onmember(this.r - p.length); + } + this.p = p.subarray(s), this.v = 0; + } + // necessary to prevent TS from using the closure value + // This allows for workerization to function correctly + Inflate.prototype.c.call(this, final); + // process concatenated GZIP + if (this.s.f && !this.s.l && !final) { + this.v = shft(this.s.p) + 9; + this.s = { i: 0 }; + this.o = new u8(0); + this.push(new u8(0), final); + } + }; + return Gunzip; +}()); +exports.Gunzip = Gunzip; +/** + * Asynchronous streaming single or multi-member GZIP decompression + */ +var AsyncGunzip = /*#__PURE__*/ (function () { + function AsyncGunzip(opts, cb) { + var _this = this; + astrmify([ + bInflt, + guze, + function () { return [astrm, Inflate, Gunzip]; } + ], this, StrmOpt.call(this, opts, cb), function (ev) { + var strm = new Gunzip(ev.data); + strm.onmember = function (offset) { return postMessage(offset); }; + onmessage = astrm(strm); + }, 9, 0, function (offset) { return _this.onmember && _this.onmember(offset); }); + } + return AsyncGunzip; +}()); +exports.AsyncGunzip = AsyncGunzip; +function gunzip(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + return cbify(data, opts, [ + bInflt, + guze, + function () { return [gunzipSync]; } + ], function (ev) { return pbf(gunzipSync(ev.data[0], ev.data[1])); }, 3, cb); +} +exports.gunzip = gunzip; +/** + * Expands GZIP data + * @param data The data to decompress + * @param opts The decompression options + * @returns The decompressed version of the data + */ +function gunzipSync(data, opts) { + var st = gzs(data); + if (st + 8 > data.length) + err(6, 'invalid gzip data'); + return inflt(data.subarray(st, -8), { i: 2 }, opts && opts.out || new u8(gzl(data)), opts && opts.dictionary); +} +exports.gunzipSync = gunzipSync; +/** + * Streaming Zlib compression + */ +var Zlib = /*#__PURE__*/ (function () { + function Zlib(opts, cb) { + this.c = adler(); + this.v = 1; + Deflate.call(this, opts, cb); + } + /** + * Pushes a chunk to be zlibbed + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + Zlib.prototype.push = function (chunk, final) { + this.c.p(chunk); + Deflate.prototype.push.call(this, chunk, final); + }; + Zlib.prototype.p = function (c, f) { + var raw = dopt(c, this.o, this.v && (this.o.dictionary ? 6 : 2), f && 4, this.s); + if (this.v) + zlh(raw, this.o), this.v = 0; + if (f) + wbytes(raw, raw.length - 4, this.c.d()); + this.ondata(raw, f); + }; + /** + * Flushes buffered uncompressed data. Useful to immediately retrieve the + * zlibbed output for small inputs. + */ + Zlib.prototype.flush = function () { + Deflate.prototype.flush.call(this); + }; + return Zlib; +}()); +exports.Zlib = Zlib; +/** + * Asynchronous streaming Zlib compression + */ +var AsyncZlib = /*#__PURE__*/ (function () { + function AsyncZlib(opts, cb) { + astrmify([ + bDflt, + zle, + function () { return [astrm, Deflate, Zlib]; } + ], this, StrmOpt.call(this, opts, cb), function (ev) { + var strm = new Zlib(ev.data); + onmessage = astrm(strm); + }, 10, 1); + } + return AsyncZlib; +}()); +exports.AsyncZlib = AsyncZlib; +function zlib(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + return cbify(data, opts, [ + bDflt, + zle, + function () { return [zlibSync]; } + ], function (ev) { return pbf(zlibSync(ev.data[0], ev.data[1])); }, 4, cb); +} +exports.zlib = zlib; +/** + * Compress data with Zlib + * @param data The data to compress + * @param opts The compression options + * @returns The zlib-compressed version of the data + */ +function zlibSync(data, opts) { + if (!opts) + opts = {}; + var a = adler(); + a.p(data); + var d = dopt(data, opts, opts.dictionary ? 6 : 2, 4); + return zlh(d, opts), wbytes(d, d.length - 4, a.d()), d; +} +exports.zlibSync = zlibSync; +/** + * Streaming Zlib decompression + */ +var Unzlib = /*#__PURE__*/ (function () { + function Unzlib(opts, cb) { + Inflate.call(this, opts, cb); + this.v = opts && opts.dictionary ? 2 : 1; + } + /** + * Pushes a chunk to be unzlibbed + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + Unzlib.prototype.push = function (chunk, final) { + Inflate.prototype.e.call(this, chunk); + if (this.v) { + if (this.p.length < 6 && !final) + return; + this.p = this.p.subarray(zls(this.p, this.v - 1)), this.v = 0; + } + if (final) { + if (this.p.length < 4) + err(6, 'invalid zlib data'); + this.p = this.p.subarray(0, -4); + } + // necessary to prevent TS from using the closure value + // This allows for workerization to function correctly + Inflate.prototype.c.call(this, final); + }; + return Unzlib; +}()); +exports.Unzlib = Unzlib; +/** + * Asynchronous streaming Zlib decompression + */ +var AsyncUnzlib = /*#__PURE__*/ (function () { + function AsyncUnzlib(opts, cb) { + astrmify([ + bInflt, + zule, + function () { return [astrm, Inflate, Unzlib]; } + ], this, StrmOpt.call(this, opts, cb), function (ev) { + var strm = new Unzlib(ev.data); + onmessage = astrm(strm); + }, 11, 0); + } + return AsyncUnzlib; +}()); +exports.AsyncUnzlib = AsyncUnzlib; +function unzlib(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + return cbify(data, opts, [ + bInflt, + zule, + function () { return [unzlibSync]; } + ], function (ev) { return pbf(unzlibSync(ev.data[0], gopt(ev.data[1]))); }, 5, cb); +} +exports.unzlib = unzlib; +/** + * Expands Zlib data + * @param data The data to decompress + * @param opts The decompression options + * @returns The decompressed version of the data + */ +function unzlibSync(data, opts) { + return inflt(data.subarray(zls(data, opts && opts.dictionary), -4), { i: 2 }, opts && opts.out, opts && opts.dictionary); +} +exports.unzlibSync = unzlibSync; +/** + * Streaming GZIP, Zlib, or raw DEFLATE decompression + */ +var Decompress = /*#__PURE__*/ (function () { + function Decompress(opts, cb) { + this.o = StrmOpt.call(this, opts, cb) || {}; + this.G = Gunzip; + this.I = Inflate; + this.Z = Unzlib; + } + // init substream + // overriden by AsyncDecompress + Decompress.prototype.i = function () { + var _this = this; + this.s.ondata = function (dat, final) { + _this.ondata(dat, final); + }; + }; + /** + * Pushes a chunk to be decompressed + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + Decompress.prototype.push = function (chunk, final) { + if (!this.ondata) + err(5); + if (!this.s) { + if (this.p && this.p.length) { + var n = new u8(this.p.length + chunk.length); + n.set(this.p), n.set(chunk, this.p.length); + } + else + this.p = chunk; + if (this.p.length > 2) { + this.s = (this.p[0] == 31 && this.p[1] == 139 && this.p[2] == 8) + ? new this.G(this.o) + : ((this.p[0] & 15) != 8 || (this.p[0] >> 4) > 7 || ((this.p[0] << 8 | this.p[1]) % 31)) + ? new this.I(this.o) + : new this.Z(this.o); + this.i(); + this.s.push(this.p, final); + this.p = null; + } + } + else + this.s.push(chunk, final); + }; + return Decompress; +}()); +exports.Decompress = Decompress; +/** + * Asynchronous streaming GZIP, Zlib, or raw DEFLATE decompression + */ +var AsyncDecompress = /*#__PURE__*/ (function () { + function AsyncDecompress(opts, cb) { + Decompress.call(this, opts, cb); + this.queuedSize = 0; + this.G = AsyncGunzip; + this.I = AsyncInflate; + this.Z = AsyncUnzlib; + } + AsyncDecompress.prototype.i = function () { + var _this = this; + this.s.ondata = function (err, dat, final) { + _this.ondata(err, dat, final); + }; + this.s.ondrain = function (size) { + _this.queuedSize -= size; + if (_this.ondrain) + _this.ondrain(size); + }; + }; + /** + * Pushes a chunk to be decompressed + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + AsyncDecompress.prototype.push = function (chunk, final) { + this.queuedSize += chunk.length; + Decompress.prototype.push.call(this, chunk, final); + }; + return AsyncDecompress; +}()); +exports.AsyncDecompress = AsyncDecompress; +function decompress(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + return (data[0] == 31 && data[1] == 139 && data[2] == 8) + ? gunzip(data, opts, cb) + : ((data[0] & 15) != 8 || (data[0] >> 4) > 7 || ((data[0] << 8 | data[1]) % 31)) + ? inflate(data, opts, cb) + : unzlib(data, opts, cb); +} +exports.decompress = decompress; +/** + * Expands compressed GZIP, Zlib, or raw DEFLATE data, automatically detecting the format + * @param data The data to decompress + * @param opts The decompression options + * @returns The decompressed version of the data + */ +function decompressSync(data, opts) { + return (data[0] == 31 && data[1] == 139 && data[2] == 8) + ? gunzipSync(data, opts) + : ((data[0] & 15) != 8 || (data[0] >> 4) > 7 || ((data[0] << 8 | data[1]) % 31)) + ? inflateSync(data, opts) + : unzlibSync(data, opts); +} +exports.decompressSync = decompressSync; +// flatten a directory structure +var fltn = function (d, p, t, o) { + for (var k in d) { + var val = d[k], n = p + k, op = o; + if (Array.isArray(val)) + op = mrg(o, val[1]), val = val[0]; + if (val instanceof u8) + t[n] = [val, op]; + else { + t[n += '/'] = [new u8(0), op]; + fltn(val, n, t, o); + } + } +}; +// text encoder +var te = typeof TextEncoder != 'undefined' && /*#__PURE__*/ new TextEncoder(); +// text decoder +var td = typeof TextDecoder != 'undefined' && /*#__PURE__*/ new TextDecoder(); +// text decoder stream +var tds = 0; +try { + td.decode(et, { stream: true }); + tds = 1; +} +catch (e) { } +// decode UTF8 +var dutf8 = function (d) { + for (var r = '', i = 0;;) { + var c = d[i++]; + var eb = (c > 127) + (c > 223) + (c > 239); + if (i + eb > d.length) + return { s: r, r: slc(d, i - 1) }; + if (!eb) + r += String.fromCharCode(c); + else if (eb == 3) { + c = ((c & 15) << 18 | (d[i++] & 63) << 12 | (d[i++] & 63) << 6 | (d[i++] & 63)) - 65536, + r += String.fromCharCode(55296 | (c >> 10), 56320 | (c & 1023)); + } + else if (eb & 1) + r += String.fromCharCode((c & 31) << 6 | (d[i++] & 63)); + else + r += String.fromCharCode((c & 15) << 12 | (d[i++] & 63) << 6 | (d[i++] & 63)); + } +}; +/** + * Streaming UTF-8 decoding + */ +var DecodeUTF8 = /*#__PURE__*/ (function () { + /** + * Creates a UTF-8 decoding stream + * @param cb The callback to call whenever data is decoded + */ + function DecodeUTF8(cb) { + this.ondata = cb; + if (tds) + this.t = new TextDecoder(); + else + this.p = et; + } + /** + * Pushes a chunk to be decoded from UTF-8 binary + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + DecodeUTF8.prototype.push = function (chunk, final) { + if (!this.ondata) + err(5); + final = !!final; + if (this.t) { + this.ondata(this.t.decode(chunk, { stream: true }), final); + if (final) { + if (this.t.decode().length) + err(8); + this.t = null; + } + return; + } + if (!this.p) + err(4); + var dat = new u8(this.p.length + chunk.length); + dat.set(this.p); + dat.set(chunk, this.p.length); + var _a = dutf8(dat), s = _a.s, r = _a.r; + if (final) { + if (r.length) + err(8); + this.p = null; + } + else + this.p = r; + this.ondata(s, final); + }; + return DecodeUTF8; +}()); +exports.DecodeUTF8 = DecodeUTF8; +/** + * Streaming UTF-8 encoding + */ +var EncodeUTF8 = /*#__PURE__*/ (function () { + /** + * Creates a UTF-8 decoding stream + * @param cb The callback to call whenever data is encoded + */ + function EncodeUTF8(cb) { + this.ondata = cb; + } + /** + * Pushes a chunk to be encoded to UTF-8 + * @param chunk The string data to push + * @param final Whether this is the last chunk + */ + EncodeUTF8.prototype.push = function (chunk, final) { + if (!this.ondata) + err(5); + if (this.d) + err(4); + this.ondata(strToU8(chunk), this.d = final || false); + }; + return EncodeUTF8; +}()); +exports.EncodeUTF8 = EncodeUTF8; +/** + * Converts a string into a Uint8Array for use with compression/decompression methods + * @param str The string to encode + * @param latin1 Whether or not to interpret the data as Latin-1. This should + * not need to be true unless decoding a binary string. + * @returns The string encoded in UTF-8/Latin-1 binary + */ +function strToU8(str, latin1) { + if (latin1) { + var ar_1 = new u8(str.length); + for (var i = 0; i < str.length; ++i) + ar_1[i] = str.charCodeAt(i); + return ar_1; + } + if (te) + return te.encode(str); + var l = str.length; + var ar = new u8(str.length + (str.length >> 1)); + var ai = 0; + var w = function (v) { ar[ai++] = v; }; + for (var i = 0; i < l; ++i) { + if (ai + 5 > ar.length) { + var n = new u8(ai + 8 + ((l - i) << 1)); + n.set(ar); + ar = n; + } + var c = str.charCodeAt(i); + if (c < 128 || latin1) + w(c); + else if (c < 2048) + w(192 | (c >> 6)), w(128 | (c & 63)); + else if (c > 55295 && c < 57344) + c = 65536 + (c & 1023 << 10) | (str.charCodeAt(++i) & 1023), + w(240 | (c >> 18)), w(128 | ((c >> 12) & 63)), w(128 | ((c >> 6) & 63)), w(128 | (c & 63)); + else + w(224 | (c >> 12)), w(128 | ((c >> 6) & 63)), w(128 | (c & 63)); + } + return slc(ar, 0, ai); +} +exports.strToU8 = strToU8; +/** + * Converts a Uint8Array to a string + * @param dat The data to decode to string + * @param latin1 Whether or not to interpret the data as Latin-1. This should + * not need to be true unless encoding to binary string. + * @returns The original UTF-8/Latin-1 string + */ +function strFromU8(dat, latin1) { + if (latin1) { + var r = ''; + for (var i = 0; i < dat.length; i += 16384) + r += String.fromCharCode.apply(null, dat.subarray(i, i + 16384)); + return r; + } + else if (td) { + return td.decode(dat); + } + else { + var _a = dutf8(dat), s = _a.s, r = _a.r; + if (r.length) + err(8); + return s; + } +} +exports.strFromU8 = strFromU8; +; +// deflate bit flag +var dbf = function (l) { return l == 1 ? 3 : l < 6 ? 2 : l == 9 ? 1 : 0; }; +// skip local zip header +var slzh = function (d, b) { return b + 30 + b2(d, b + 26) + b2(d, b + 28); }; +// read zip header +var zh = function (d, b, z) { + var fnl = b2(d, b + 28), fn = strFromU8(d.subarray(b + 46, b + 46 + fnl), !(b2(d, b + 8) & 2048)), es = b + 46 + fnl, bs = b4(d, b + 20); + var _a = z && bs == 4294967295 ? z64e(d, es) : [bs, b4(d, b + 24), b4(d, b + 42)], sc = _a[0], su = _a[1], off = _a[2]; + return [b2(d, b + 10), sc, su, fn, es + b2(d, b + 30) + b2(d, b + 32), off]; +}; +// read zip64 extra field +var z64e = function (d, b) { + for (; b2(d, b) != 1; b += 4 + b2(d, b + 2)) + ; + return [b8(d, b + 12), b8(d, b + 4), b8(d, b + 20)]; +}; +// extra field length +var exfl = function (ex) { + var le = 0; + if (ex) { + for (var k in ex) { + var l = ex[k].length; + if (l > 65535) + err(9); + le += l + 4; + } + } + return le; +}; +// write zip header +var wzh = function (d, b, f, fn, u, c, ce, co) { + var fl = fn.length, ex = f.extra, col = co && co.length; + var exl = exfl(ex); + wbytes(d, b, ce != null ? 0x2014B50 : 0x4034B50), b += 4; + if (ce != null) + d[b++] = 20, d[b++] = f.os; + d[b] = 20, b += 2; // spec compliance? what's that? + d[b++] = (f.flag << 1) | (c < 0 && 8), d[b++] = u && 8; + d[b++] = f.compression & 255, d[b++] = f.compression >> 8; + var dt = new Date(f.mtime == null ? Date.now() : f.mtime), y = dt.getFullYear() - 1980; + if (y < 0 || y > 119) + err(10); + wbytes(d, b, (y << 25) | ((dt.getMonth() + 1) << 21) | (dt.getDate() << 16) | (dt.getHours() << 11) | (dt.getMinutes() << 5) | (dt.getSeconds() >> 1)), b += 4; + if (c != -1) { + wbytes(d, b, f.crc); + wbytes(d, b + 4, c < 0 ? -c - 2 : c); + wbytes(d, b + 8, f.size); + } + wbytes(d, b + 12, fl); + wbytes(d, b + 14, exl), b += 16; + if (ce != null) { + wbytes(d, b, col); + wbytes(d, b + 6, f.attrs); + wbytes(d, b + 10, ce), b += 14; + } + d.set(fn, b); + b += fl; + if (exl) { + for (var k in ex) { + var exf = ex[k], l = exf.length; + wbytes(d, b, +k); + wbytes(d, b + 2, l); + d.set(exf, b + 4), b += 4 + l; + } + } + if (col) + d.set(co, b), b += col; + return b; +}; +// write zip footer (end of central directory) +var wzf = function (o, b, c, d, e) { + wbytes(o, b, 0x6054B50); // skip disk + wbytes(o, b + 8, c); + wbytes(o, b + 10, c); + wbytes(o, b + 12, d); + wbytes(o, b + 16, e); +}; +/** + * A pass-through stream to keep data uncompressed in a ZIP archive. + */ +var ZipPassThrough = /*#__PURE__*/ (function () { + /** + * Creates a pass-through stream that can be added to ZIP archives + * @param filename The filename to associate with this data stream + */ + function ZipPassThrough(filename) { + this.filename = filename; + this.c = crc(); + this.size = 0; + this.compression = 0; + } + /** + * Processes a chunk and pushes to the output stream. You can override this + * method in a subclass for custom behavior, but by default this passes + * the data through. You must call this.ondata(err, chunk, final) at some + * point in this method. + * @param chunk The chunk to process + * @param final Whether this is the last chunk + */ + ZipPassThrough.prototype.process = function (chunk, final) { + this.ondata(null, chunk, final); + }; + /** + * Pushes a chunk to be added. If you are subclassing this with a custom + * compression algorithm, note that you must push data from the source + * file only, pre-compression. + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + ZipPassThrough.prototype.push = function (chunk, final) { + if (!this.ondata) + err(5); + this.c.p(chunk); + this.size += chunk.length; + if (final) + this.crc = this.c.d(); + this.process(chunk, final || false); + }; + return ZipPassThrough; +}()); +exports.ZipPassThrough = ZipPassThrough; +// I don't extend because TypeScript extension adds 1kB of runtime bloat +/** + * Streaming DEFLATE compression for ZIP archives. Prefer using AsyncZipDeflate + * for better performance + */ +var ZipDeflate = /*#__PURE__*/ (function () { + /** + * Creates a DEFLATE stream that can be added to ZIP archives + * @param filename The filename to associate with this data stream + * @param opts The compression options + */ + function ZipDeflate(filename, opts) { + var _this = this; + if (!opts) + opts = {}; + ZipPassThrough.call(this, filename); + this.d = new Deflate(opts, function (dat, final) { + _this.ondata(null, dat, final); + }); + this.compression = 8; + this.flag = dbf(opts.level); + } + ZipDeflate.prototype.process = function (chunk, final) { + try { + this.d.push(chunk, final); + } + catch (e) { + this.ondata(e, null, final); + } + }; + /** + * Pushes a chunk to be deflated + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + ZipDeflate.prototype.push = function (chunk, final) { + ZipPassThrough.prototype.push.call(this, chunk, final); + }; + return ZipDeflate; +}()); +exports.ZipDeflate = ZipDeflate; +/** + * Asynchronous streaming DEFLATE compression for ZIP archives + */ +var AsyncZipDeflate = /*#__PURE__*/ (function () { + /** + * Creates an asynchronous DEFLATE stream that can be added to ZIP archives + * @param filename The filename to associate with this data stream + * @param opts The compression options + */ + function AsyncZipDeflate(filename, opts) { + var _this = this; + if (!opts) + opts = {}; + ZipPassThrough.call(this, filename); + this.d = new AsyncDeflate(opts, function (err, dat, final) { + _this.ondata(err, dat, final); + }); + this.compression = 8; + this.flag = dbf(opts.level); + this.terminate = this.d.terminate; + } + AsyncZipDeflate.prototype.process = function (chunk, final) { + this.d.push(chunk, final); + }; + /** + * Pushes a chunk to be deflated + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + AsyncZipDeflate.prototype.push = function (chunk, final) { + ZipPassThrough.prototype.push.call(this, chunk, final); + }; + return AsyncZipDeflate; +}()); +exports.AsyncZipDeflate = AsyncZipDeflate; +// TODO: Better tree shaking +/** + * A zippable archive to which files can incrementally be added + */ +var Zip = /*#__PURE__*/ (function () { + /** + * Creates an empty ZIP archive to which files can be added + * @param cb The callback to call whenever data for the generated ZIP archive + * is available + */ + function Zip(cb) { + this.ondata = cb; + this.u = []; + this.d = 1; + } + /** + * Adds a file to the ZIP archive + * @param file The file stream to add + */ + Zip.prototype.add = function (file) { + var _this = this; + if (!this.ondata) + err(5); + // finishing or finished + if (this.d & 2) + this.ondata(err(4 + (this.d & 1) * 8, 0, 1), null, false); + else { + var f = strToU8(file.filename), fl_1 = f.length; + var com = file.comment, o = com && strToU8(com); + var u = fl_1 != file.filename.length || (o && (com.length != o.length)); + var hl_1 = fl_1 + exfl(file.extra) + 30; + if (fl_1 > 65535) + this.ondata(err(11, 0, 1), null, false); + var header = new u8(hl_1); + wzh(header, 0, file, f, u, -1); + var chks_1 = [header]; + var pAll_1 = function () { + for (var _i = 0, chks_2 = chks_1; _i < chks_2.length; _i++) { + var chk = chks_2[_i]; + _this.ondata(null, chk, false); + } + chks_1 = []; + }; + var tr_1 = this.d; + this.d = 0; + var ind_1 = this.u.length; + var uf_1 = mrg(file, { + f: f, + u: u, + o: o, + t: function () { + if (file.terminate) + file.terminate(); + }, + r: function () { + pAll_1(); + if (tr_1) { + var nxt = _this.u[ind_1 + 1]; + if (nxt) + nxt.r(); + else + _this.d = 1; + } + tr_1 = 1; + } + }); + var cl_1 = 0; + file.ondata = function (err, dat, final) { + if (err) { + _this.ondata(err, dat, final); + _this.terminate(); + } + else { + cl_1 += dat.length; + chks_1.push(dat); + if (final) { + var dd = new u8(16); + wbytes(dd, 0, 0x8074B50); + wbytes(dd, 4, file.crc); + wbytes(dd, 8, cl_1); + wbytes(dd, 12, file.size); + chks_1.push(dd); + uf_1.c = cl_1, uf_1.b = hl_1 + cl_1 + 16, uf_1.crc = file.crc, uf_1.size = file.size; + if (tr_1) + uf_1.r(); + tr_1 = 1; + } + else if (tr_1) + pAll_1(); + } + }; + this.u.push(uf_1); + } + }; + /** + * Ends the process of adding files and prepares to emit the final chunks. + * This *must* be called after adding all desired files for the resulting + * ZIP file to work properly. + */ + Zip.prototype.end = function () { + var _this = this; + if (this.d & 2) { + this.ondata(err(4 + (this.d & 1) * 8, 0, 1), null, true); + return; + } + if (this.d) + this.e(); + else + this.u.push({ + r: function () { + if (!(_this.d & 1)) + return; + _this.u.splice(-1, 1); + _this.e(); + }, + t: function () { } + }); + this.d = 3; + }; + Zip.prototype.e = function () { + var bt = 0, l = 0, tl = 0; + for (var _i = 0, _a = this.u; _i < _a.length; _i++) { + var f = _a[_i]; + tl += 46 + f.f.length + exfl(f.extra) + (f.o ? f.o.length : 0); + } + var out = new u8(tl + 22); + for (var _b = 0, _c = this.u; _b < _c.length; _b++) { + var f = _c[_b]; + wzh(out, bt, f, f.f, f.u, -f.c - 2, l, f.o); + bt += 46 + f.f.length + exfl(f.extra) + (f.o ? f.o.length : 0), l += f.b; + } + wzf(out, bt, this.u.length, tl, l); + this.ondata(null, out, true); + this.d = 2; + }; + /** + * A method to terminate any internal workers used by the stream. Subsequent + * calls to add() will fail. + */ + Zip.prototype.terminate = function () { + for (var _i = 0, _a = this.u; _i < _a.length; _i++) { + var f = _a[_i]; + f.t(); + } + this.d = 2; + }; + return Zip; +}()); +exports.Zip = Zip; +function zip(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + var r = {}; + fltn(data, '', r, opts); + var k = Object.keys(r); + var lft = k.length, o = 0, tot = 0; + var slft = lft, files = new Array(lft); + var term = []; + var tAll = function () { + for (var i = 0; i < term.length; ++i) + term[i](); + }; + var cbd = function (a, b) { + mt(function () { cb(a, b); }); + }; + mt(function () { cbd = cb; }); + var cbf = function () { + var out = new u8(tot + 22), oe = o, cdl = tot - o; + tot = 0; + for (var i = 0; i < slft; ++i) { + var f = files[i]; + try { + var l = f.c.length; + wzh(out, tot, f, f.f, f.u, l); + var badd = 30 + f.f.length + exfl(f.extra); + var loc = tot + badd; + out.set(f.c, loc); + wzh(out, o, f, f.f, f.u, l, tot, f.m), o += 16 + badd + (f.m ? f.m.length : 0), tot = loc + l; + } + catch (e) { + return cbd(e, null); + } + } + wzf(out, o, files.length, cdl, oe); + cbd(null, out); + }; + if (!lft) + cbf(); + var _loop_1 = function (i) { + var fn = k[i]; + var _a = r[fn], file = _a[0], p = _a[1]; + var c = crc(), size = file.length; + c.p(file); + var f = strToU8(fn), s = f.length; + var com = p.comment, m = com && strToU8(com), ms = m && m.length; + var exl = exfl(p.extra); + var compression = p.level == 0 ? 0 : 8; + var cbl = function (e, d) { + if (e) { + tAll(); + cbd(e, null); + } + else { + var l = d.length; + files[i] = mrg(p, { + size: size, + crc: c.d(), + c: d, + f: f, + m: m, + u: s != fn.length || (m && (com.length != ms)), + compression: compression + }); + o += 30 + s + exl + l; + tot += 76 + 2 * (s + exl) + (ms || 0) + l; + if (!--lft) + cbf(); + } + }; + if (s > 65535) + cbl(err(11, 0, 1), null); + if (!compression) + cbl(null, file); + else if (size < 160000) { + try { + cbl(null, deflateSync(file, p)); + } + catch (e) { + cbl(e, null); + } + } + else + term.push(deflate(file, p, cbl)); + }; + // Cannot use lft because it can decrease + for (var i = 0; i < slft; ++i) { + _loop_1(i); + } + return tAll; +} +exports.zip = zip; +/** + * Synchronously creates a ZIP file. Prefer using `zip` for better performance + * with more than one file. + * @param data The directory structure for the ZIP archive + * @param opts The main options, merged with per-file options + * @returns The generated ZIP archive + */ +function zipSync(data, opts) { + if (!opts) + opts = {}; + var r = {}; + var files = []; + fltn(data, '', r, opts); + var o = 0; + var tot = 0; + for (var fn in r) { + var _a = r[fn], file = _a[0], p = _a[1]; + var compression = p.level == 0 ? 0 : 8; + var f = strToU8(fn), s = f.length; + var com = p.comment, m = com && strToU8(com), ms = m && m.length; + var exl = exfl(p.extra); + if (s > 65535) + err(11); + var d = compression ? deflateSync(file, p) : file, l = d.length; + var c = crc(); + c.p(file); + files.push(mrg(p, { + size: file.length, + crc: c.d(), + c: d, + f: f, + m: m, + u: s != fn.length || (m && (com.length != ms)), + o: o, + compression: compression + })); + o += 30 + s + exl + l; + tot += 76 + 2 * (s + exl) + (ms || 0) + l; + } + var out = new u8(tot + 22), oe = o, cdl = tot - o; + for (var i = 0; i < files.length; ++i) { + var f = files[i]; + wzh(out, f.o, f, f.f, f.u, f.c.length); + var badd = 30 + f.f.length + exfl(f.extra); + out.set(f.c, f.o + badd); + wzh(out, o, f, f.f, f.u, f.c.length, f.o, f.m), o += 16 + badd + (f.m ? f.m.length : 0); + } + wzf(out, o, files.length, cdl, oe); + return out; +} +exports.zipSync = zipSync; +/** + * Streaming pass-through decompression for ZIP archives + */ +var UnzipPassThrough = /*#__PURE__*/ (function () { + function UnzipPassThrough() { + } + UnzipPassThrough.prototype.push = function (data, final) { + this.ondata(null, data, final); + }; + UnzipPassThrough.compression = 0; + return UnzipPassThrough; +}()); +exports.UnzipPassThrough = UnzipPassThrough; +/** + * Streaming DEFLATE decompression for ZIP archives. Prefer AsyncZipInflate for + * better performance. + */ +var UnzipInflate = /*#__PURE__*/ (function () { + /** + * Creates a DEFLATE decompression that can be used in ZIP archives + */ + function UnzipInflate() { + var _this = this; + this.i = new Inflate(function (dat, final) { + _this.ondata(null, dat, final); + }); + } + UnzipInflate.prototype.push = function (data, final) { + try { + this.i.push(data, final); + } + catch (e) { + this.ondata(e, null, final); + } + }; + UnzipInflate.compression = 8; + return UnzipInflate; +}()); +exports.UnzipInflate = UnzipInflate; +/** + * Asynchronous streaming DEFLATE decompression for ZIP archives + */ +var AsyncUnzipInflate = /*#__PURE__*/ (function () { + /** + * Creates a DEFLATE decompression that can be used in ZIP archives + */ + function AsyncUnzipInflate(_, sz) { + var _this = this; + if (sz < 320000) { + this.i = new Inflate(function (dat, final) { + _this.ondata(null, dat, final); + }); + } + else { + this.i = new AsyncInflate(function (err, dat, final) { + _this.ondata(err, dat, final); + }); + this.terminate = this.i.terminate; + } + } + AsyncUnzipInflate.prototype.push = function (data, final) { + if (this.i.terminate) + data = slc(data, 0); + this.i.push(data, final); + }; + AsyncUnzipInflate.compression = 8; + return AsyncUnzipInflate; +}()); +exports.AsyncUnzipInflate = AsyncUnzipInflate; +/** + * A ZIP archive decompression stream that emits files as they are discovered + */ +var Unzip = /*#__PURE__*/ (function () { + /** + * Creates a ZIP decompression stream + * @param cb The callback to call whenever a file in the ZIP archive is found + */ + function Unzip(cb) { + this.onfile = cb; + this.k = []; + this.o = { + 0: UnzipPassThrough + }; + this.p = et; + } + /** + * Pushes a chunk to be unzipped + * @param chunk The chunk to push + * @param final Whether this is the last chunk + */ + Unzip.prototype.push = function (chunk, final) { + var _this = this; + if (!this.onfile) + err(5); + if (!this.p) + err(4); + if (this.c > 0) { + var len = Math.min(this.c, chunk.length); + var toAdd = chunk.subarray(0, len); + this.c -= len; + if (this.d) + this.d.push(toAdd, !this.c); + else + this.k[0].push(toAdd); + chunk = chunk.subarray(len); + if (chunk.length) + return this.push(chunk, final); + } + else { + var f = 0, i = 0, is = void 0, buf = void 0; + if (!this.p.length) + buf = chunk; + else if (!chunk.length) + buf = this.p; + else { + buf = new u8(this.p.length + chunk.length); + buf.set(this.p), buf.set(chunk, this.p.length); + } + var l = buf.length, oc = this.c, add = oc && this.d; + var _loop_2 = function () { + var _a; + var sig = b4(buf, i); + if (sig == 0x4034B50) { + f = 1, is = i; + this_1.d = null; + this_1.c = 0; + var bf = b2(buf, i + 6), cmp_1 = b2(buf, i + 8), u = bf & 2048, dd = bf & 8, fnl = b2(buf, i + 26), es = b2(buf, i + 28); + if (l > i + 30 + fnl + es) { + var chks_3 = []; + this_1.k.unshift(chks_3); + f = 2; + var sc_1 = b4(buf, i + 18), su_1 = b4(buf, i + 22); + var fn_1 = strFromU8(buf.subarray(i + 30, i += 30 + fnl), !u); + if (sc_1 == 4294967295) { + _a = dd ? [-2] : z64e(buf, i), sc_1 = _a[0], su_1 = _a[1]; + } + else if (dd) + sc_1 = -1; + i += es; + this_1.c = sc_1; + var d_1; + var file_1 = { + name: fn_1, + compression: cmp_1, + start: function () { + if (!file_1.ondata) + err(5); + if (!sc_1) + file_1.ondata(null, et, true); + else { + var ctr = _this.o[cmp_1]; + if (!ctr) + file_1.ondata(err(14, 'unknown compression type ' + cmp_1, 1), null, false); + d_1 = sc_1 < 0 ? new ctr(fn_1) : new ctr(fn_1, sc_1, su_1); + d_1.ondata = function (err, dat, final) { file_1.ondata(err, dat, final); }; + for (var _i = 0, chks_4 = chks_3; _i < chks_4.length; _i++) { + var dat = chks_4[_i]; + d_1.push(dat, false); + } + if (_this.k[0] == chks_3 && _this.c) + _this.d = d_1; + else + d_1.push(et, true); + } + }, + terminate: function () { + if (d_1 && d_1.terminate) + d_1.terminate(); + } + }; + if (sc_1 >= 0) + file_1.size = sc_1, file_1.originalSize = su_1; + this_1.onfile(file_1); + } + return "break"; + } + else if (oc) { + if (sig == 0x8074B50) { + is = i += 12 + (oc == -2 && 8), f = 3, this_1.c = 0; + return "break"; + } + else if (sig == 0x2014B50) { + is = i -= 4, f = 3, this_1.c = 0; + return "break"; + } + } + }; + var this_1 = this; + for (; i < l - 4; ++i) { + var state_1 = _loop_2(); + if (state_1 === "break") + break; + } + this.p = et; + if (oc < 0) { + var dat = f ? buf.subarray(0, is - 12 - (oc == -2 && 8) - (b4(buf, is - 16) == 0x8074B50 && 4)) : buf.subarray(0, i); + if (add) + add.push(dat, !!f); + else + this.k[+(f == 2)].push(dat); + } + if (f & 2) + return this.push(buf.subarray(i), final); + this.p = buf.subarray(i); + } + if (final) { + if (this.c) + err(13); + this.p = null; + } + }; + /** + * Registers a decoder with the stream, allowing for files compressed with + * the compression type provided to be expanded correctly + * @param decoder The decoder constructor + */ + Unzip.prototype.register = function (decoder) { + this.o[decoder.compression] = decoder; + }; + return Unzip; +}()); +exports.Unzip = Unzip; +var mt = typeof queueMicrotask == 'function' ? queueMicrotask : typeof setTimeout == 'function' ? setTimeout : function (fn) { fn(); }; +function unzip(data, opts, cb) { + if (!cb) + cb = opts, opts = {}; + if (typeof cb != 'function') + err(7); + var term = []; + var tAll = function () { + for (var i = 0; i < term.length; ++i) + term[i](); + }; + var files = {}; + var cbd = function (a, b) { + mt(function () { cb(a, b); }); + }; + mt(function () { cbd = cb; }); + var e = data.length - 22; + for (; b4(data, e) != 0x6054B50; --e) { + if (!e || data.length - e > 65558) { + cbd(err(13, 0, 1), null); + return tAll; + } + } + ; + var lft = b2(data, e + 8); + if (lft) { + var c = lft; + var o = b4(data, e + 16); + var z = o == 4294967295 || c == 65535; + if (z) { + var ze = b4(data, e - 12); + z = b4(data, ze) == 0x6064B50; + if (z) { + c = lft = b4(data, ze + 32); + o = b4(data, ze + 48); + } + } + var fltr = opts && opts.filter; + var _loop_3 = function (i) { + var _a = zh(data, o, z), c_1 = _a[0], sc = _a[1], su = _a[2], fn = _a[3], no = _a[4], off = _a[5], b = slzh(data, off); + o = no; + var cbl = function (e, d) { + if (e) { + tAll(); + cbd(e, null); + } + else { + if (d) + files[fn] = d; + if (!--lft) + cbd(null, files); + } + }; + if (!fltr || fltr({ + name: fn, + size: sc, + originalSize: su, + compression: c_1 + })) { + if (!c_1) + cbl(null, slc(data, b, b + sc)); + else if (c_1 == 8) { + var infl = data.subarray(b, b + sc); + // Synchronously decompress under 512KB, or barely-compressed data + if (su < 524288 || sc > 0.8 * su) { + try { + cbl(null, inflateSync(infl, { out: new u8(su) })); + } + catch (e) { + cbl(e, null); + } + } + else + term.push(inflate(infl, { size: su }, cbl)); + } + else + cbl(err(14, 'unknown compression type ' + c_1, 1), null); + } + else + cbl(null, null); + }; + for (var i = 0; i < c; ++i) { + _loop_3(i); + } + } + else + cbd(null, {}); + return tAll; +} +exports.unzip = unzip; +/** + * Synchronously decompresses a ZIP archive. Prefer using `unzip` for better + * performance with more than one file. + * @param data The raw compressed ZIP file + * @param opts The ZIP extraction options + * @returns The decompressed files + */ +function unzipSync(data, opts) { + var files = {}; + var e = data.length - 22; + for (; b4(data, e) != 0x6054B50; --e) { + if (!e || data.length - e > 65558) + err(13); + } + ; + var c = b2(data, e + 8); + if (!c) + return {}; + var o = b4(data, e + 16); + var z = o == 4294967295 || c == 65535; + if (z) { + var ze = b4(data, e - 12); + z = b4(data, ze) == 0x6064B50; + if (z) { + c = b4(data, ze + 32); + o = b4(data, ze + 48); + } + } + var fltr = opts && opts.filter; + for (var i = 0; i < c; ++i) { + var _a = zh(data, o, z), c_2 = _a[0], sc = _a[1], su = _a[2], fn = _a[3], no = _a[4], off = _a[5], b = slzh(data, off); + o = no; + if (!fltr || fltr({ + name: fn, + size: sc, + originalSize: su, + compression: c_2 + })) { + if (!c_2) + files[fn] = slc(data, b, b + sc); + else if (c_2 == 8) + files[fn] = inflateSync(data.subarray(b, b + sc), { out: new u8(su) }); + else + err(14, 'unknown compression type ' + c_2); + } + } + return files; +} +exports.unzipSync = unzipSync; + +},{"./node-worker.cjs":4}],4:[function(require,module,exports){ +"use strict"; +var ch2 = {}; +exports.default = (function (c, id, msg, transfer, cb) { + var w = new Worker(ch2[id] || (ch2[id] = URL.createObjectURL(new Blob([ + c + ';addEventListener("error",function(e){e=e.error;postMessage({$e$:[e.message,e.code,e.stack]})})' + ], { type: 'text/javascript' })))); + w.onmessage = function (e) { + var d = e.data, ed = d.$e$; + if (ed) { + var err = new Error(ed[0]); + err['code'] = ed[1]; + err.stack = ed[2]; + cb(err, null); + } + else + cb(null, d); + }; + w.postMessage(msg, transfer); + return w; +}); + +},{}],5:[function(require,module,exports){ +// shim for using process in browser +var process = module.exports = {}; + +// cached from whatever global is present so that test runners that stub it +// don't break things. But we need to wrap it in a try catch in case it is +// wrapped in strict mode code which doesn't define any globals. It's inside a +// function because try/catches deoptimize in certain engines. + +var cachedSetTimeout; +var cachedClearTimeout; + +function defaultSetTimout() { + throw new Error('setTimeout has not been defined'); +} +function defaultClearTimeout () { + throw new Error('clearTimeout has not been defined'); +} +(function () { + try { + if (typeof setTimeout === 'function') { + cachedSetTimeout = setTimeout; + } else { + cachedSetTimeout = defaultSetTimout; + } + } catch (e) { + cachedSetTimeout = defaultSetTimout; + } + try { + if (typeof clearTimeout === 'function') { + cachedClearTimeout = clearTimeout; + } else { + cachedClearTimeout = defaultClearTimeout; + } + } catch (e) { + cachedClearTimeout = defaultClearTimeout; + } +} ()) +function runTimeout(fun) { + if (cachedSetTimeout === setTimeout) { + //normal enviroments in sane situations + return setTimeout(fun, 0); + } + // if setTimeout wasn't available but was latter defined + if ((cachedSetTimeout === defaultSetTimout || !cachedSetTimeout) && setTimeout) { + cachedSetTimeout = setTimeout; + return setTimeout(fun, 0); + } + try { + // when when somebody has screwed with setTimeout but no I.E. maddness + return cachedSetTimeout(fun, 0); + } catch(e){ + try { + // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally + return cachedSetTimeout.call(null, fun, 0); + } catch(e){ + // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error + return cachedSetTimeout.call(this, fun, 0); + } + } + + +} +function runClearTimeout(marker) { + if (cachedClearTimeout === clearTimeout) { + //normal enviroments in sane situations + return clearTimeout(marker); + } + // if clearTimeout wasn't available but was latter defined + if ((cachedClearTimeout === defaultClearTimeout || !cachedClearTimeout) && clearTimeout) { + cachedClearTimeout = clearTimeout; + return clearTimeout(marker); + } + try { + // when when somebody has screwed with setTimeout but no I.E. maddness + return cachedClearTimeout(marker); + } catch (e){ + try { + // When we are in I.E. but the script has been evaled so I.E. doesn't trust the global object when called normally + return cachedClearTimeout.call(null, marker); + } catch (e){ + // same as above but when it's a version of I.E. that must have the global object for 'this', hopfully our context correct otherwise it will throw a global error. + // Some versions of I.E. have different rules for clearTimeout vs setTimeout + return cachedClearTimeout.call(this, marker); + } + } + + + +} +var queue = []; +var draining = false; +var currentQueue; +var queueIndex = -1; - - -} -var queue = []; -var draining = false; -var currentQueue; -var queueIndex = -1; - function cleanUpNextTick() { if (!draining || !currentQueue) { return; @@ -6804,35 +9285,86 @@ process.chdir = function (dir) { }; process.umask = function() { return 0; }; -},{}],5:[function(require,module,exports){ -/** @license zlib.js 2012 - imaya [ https://github.com/imaya/zlib.js ] The MIT License */(function() {'use strict';function n(e){throw e;}var p=void 0,aa=this;function t(e,b){var d=e.split("."),c=aa;!(d[0]in c)&&c.execScript&&c.execScript("var "+d[0]);for(var a;d.length&&(a=d.shift());)!d.length&&b!==p?c[a]=b:c=c[a]?c[a]:c[a]={}};var x="undefined"!==typeof Uint8Array&&"undefined"!==typeof Uint16Array&&"undefined"!==typeof Uint32Array&&"undefined"!==typeof DataView;new (x?Uint8Array:Array)(256);var y;for(y=0;256>y;++y)for(var A=y,ba=7,A=A>>>1;A;A>>>=1)--ba;function B(e,b,d){var c,a="number"===typeof b?b:b=0,f="number"===typeof d?d:e.length;c=-1;for(a=f&7;a--;++b)c=c>>>8^C[(c^e[b])&255];for(a=f>>3;a--;b+=8)c=c>>>8^C[(c^e[b])&255],c=c>>>8^C[(c^e[b+1])&255],c=c>>>8^C[(c^e[b+2])&255],c=c>>>8^C[(c^e[b+3])&255],c=c>>>8^C[(c^e[b+4])&255],c=c>>>8^C[(c^e[b+5])&255],c=c>>>8^C[(c^e[b+6])&255],c=c>>>8^C[(c^e[b+7])&255];return(c^4294967295)>>>0} -var D=[0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125561021,3887607047,2428444049,498536548,1789927666,4089016648,2227061214,450548861,1843258603,4107580753,2211677639,325883990,1684777152,4251122042,2321926636,335633487,1661365465,4195302755,2366115317,997073096,1281953886,3579855332,2724688242,1006888145,1258607687,3524101629,2768942443,901097722,1119000684,3686517206,2898065728,853044451,1172266101,3705015759, -2882616665,651767980,1373503546,3369554304,3218104598,565507253,1454621731,3485111705,3099436303,671266974,1594198024,3322730930,2970347812,795835527,1483230225,3244367275,3060149565,1994146192,31158534,2563907772,4023717930,1907459465,112637215,2680153253,3904427059,2013776290,251722036,2517215374,3775830040,2137656763,141376813,2439277719,3865271297,1802195444,476864866,2238001368,4066508878,1812370925,453092731,2181625025,4111451223,1706088902,314042704,2344532202,4240017532,1658658271,366619977, -2362670323,4224994405,1303535960,984961486,2747007092,3569037538,1256170817,1037604311,2765210733,3554079995,1131014506,879679996,2909243462,3663771856,1141124467,855842277,2852801631,3708648649,1342533948,654459306,3188396048,3373015174,1466479909,544179635,3110523913,3462522015,1591671054,702138776,2966460450,3352799412,1504918807,783551873,3082640443,3233442989,3988292384,2596254646,62317068,1957810842,3939845945,2647816111,81470997,1943803523,3814918930,2489596804,225274430,2053790376,3826175755, -2466906013,167816743,2097651377,4027552580,2265490386,503444072,1762050814,4150417245,2154129355,426522225,1852507879,4275313526,2312317920,282753626,1742555852,4189708143,2394877945,397917763,1622183637,3604390888,2714866558,953729732,1340076626,3518719985,2797360999,1068828381,1219638859,3624741850,2936675148,906185462,1090812512,3747672003,2825379669,829329135,1181335161,3412177804,3160834842,628085408,1382605366,3423369109,3138078467,570562233,1426400815,3317316542,2998733608,733239954,1555261956, -3268935591,3050360625,752459403,1541320221,2607071920,3965973030,1969922972,40735498,2617837225,3943577151,1913087877,83908371,2512341634,3803740692,2075208622,213261112,2463272603,3855990285,2094854071,198958881,2262029012,4057260610,1759359992,534414190,2176718541,4139329115,1873836001,414664567,2282248934,4279200368,1711684554,285281116,2405801727,4167216745,1634467795,376229701,2685067896,3608007406,1308918612,956543938,2808555105,3495958263,1231636301,1047427035,2932959818,3654703836,1088359270, -936918E3,2847714899,3736837829,1202900863,817233897,3183342108,3401237130,1404277552,615818150,3134207493,3453421203,1423857449,601450431,3009837614,3294710456,1567103746,711928724,3020668471,3272380065,1510334235,755167117],C=x?new Uint32Array(D):D;function E(){}E.prototype.getName=function(){return this.name};E.prototype.getData=function(){return this.data};E.prototype.G=function(){return this.H};function G(e){var b=e.length,d=0,c=Number.POSITIVE_INFINITY,a,f,k,l,m,r,q,g,h,v;for(g=0;gd&&(d=e[g]),e[g]>=1;v=k<<16|g;for(h=r;hK;K++)switch(!0){case 143>=K:J.push([K+48,8]);break;case 255>=K:J.push([K-144+400,9]);break;case 279>=K:J.push([K-256+0,7]);break;case 287>=K:J.push([K-280+192,8]);break;default:n("invalid literal: "+K)} -var ca=function(){function e(a){switch(!0){case 3===a:return[257,a-3,0];case 4===a:return[258,a-4,0];case 5===a:return[259,a-5,0];case 6===a:return[260,a-6,0];case 7===a:return[261,a-7,0];case 8===a:return[262,a-8,0];case 9===a:return[263,a-9,0];case 10===a:return[264,a-10,0];case 12>=a:return[265,a-11,1];case 14>=a:return[266,a-13,1];case 16>=a:return[267,a-15,1];case 18>=a:return[268,a-17,1];case 22>=a:return[269,a-19,2];case 26>=a:return[270,a-23,2];case 30>=a:return[271,a-27,2];case 34>=a:return[272, -a-31,2];case 42>=a:return[273,a-35,3];case 50>=a:return[274,a-43,3];case 58>=a:return[275,a-51,3];case 66>=a:return[276,a-59,3];case 82>=a:return[277,a-67,4];case 98>=a:return[278,a-83,4];case 114>=a:return[279,a-99,4];case 130>=a:return[280,a-115,4];case 162>=a:return[281,a-131,5];case 194>=a:return[282,a-163,5];case 226>=a:return[283,a-195,5];case 257>=a:return[284,a-227,5];case 258===a:return[285,a-258,0];default:n("invalid length: "+a)}}var b=[],d,c;for(d=3;258>=d;d++)c=e(d),b[d]=c[2]<<24|c[1]<< -16|c[0];return b}();x&&new Uint32Array(ca);function L(e,b){this.i=[];this.j=32768;this.d=this.f=this.c=this.n=0;this.input=x?new Uint8Array(e):e;this.o=!1;this.k=M;this.w=!1;if(b||!(b={}))b.index&&(this.c=b.index),b.bufferSize&&(this.j=b.bufferSize),b.bufferType&&(this.k=b.bufferType),b.resize&&(this.w=b.resize);switch(this.k){case N:this.a=32768;this.b=new (x?Uint8Array:Array)(32768+this.j+258);break;case M:this.a=0;this.b=new (x?Uint8Array:Array)(this.j);this.e=this.D;this.q=this.A;this.l=this.C;break;default:n(Error("invalid inflate mode"))}} -var N=0,M=1; -L.prototype.g=function(){for(;!this.o;){var e=P(this,3);e&1&&(this.o=!0);e>>>=1;switch(e){case 0:var b=this.input,d=this.c,c=this.b,a=this.a,f=b.length,k=p,l=p,m=c.length,r=p;this.d=this.f=0;d+1>=f&&n(Error("invalid uncompressed block header: LEN"));k=b[d++]|b[d++]<<8;d+1>=f&&n(Error("invalid uncompressed block header: NLEN"));l=b[d++]|b[d++]<<8;k===~l&&n(Error("invalid uncompressed block header: length verify"));d+k>b.length&&n(Error("input buffer is broken"));switch(this.k){case N:for(;a+k>c.length;){r= -m-a;k-=r;if(x)c.set(b.subarray(d,d+r),a),a+=r,d+=r;else for(;r--;)c[a++]=b[d++];this.a=a;c=this.e();a=this.a}break;case M:for(;a+k>c.length;)c=this.e({t:2});break;default:n(Error("invalid inflate mode"))}if(x)c.set(b.subarray(d,d+k),a),a+=k,d+=k;else for(;k--;)c[a++]=b[d++];this.c=d;this.a=a;this.b=c;break;case 1:this.l(da,ea);break;case 2:for(var q=P(this,5)+257,g=P(this,5)+1,h=P(this,4)+4,v=new (x?Uint8Array:Array)(Q.length),s=p,F=p,H=p,w=p,z=p,O=p,I=p,u=p,Z=p,u=0;u=W?8:255>=W?9:279>=W?7:8;var da=G(V),X=new (x?Uint8Array:Array)(30),Y,ma;Y=0;for(ma=X.length;Y=k&&n(Error("input buffer is broken")),d|=a[f++]<>>b;e.d=c-b;e.c=f;return l} -function R(e,b){for(var d=e.f,c=e.d,a=e.input,f=e.c,k=a.length,l=b[0],m=b[1],r,q;c=k);)d|=a[f++]<>>16;q>c&&n(Error("invalid code length: "+q));e.f=d>>q;e.d=c-q;e.c=f;return r&65535} -L.prototype.l=function(e,b){var d=this.b,c=this.a;this.r=e;for(var a=d.length-258,f,k,l,m;256!==(f=R(this,e));)if(256>f)c>=a&&(this.a=c,d=this.e(),c=this.a),d[c++]=f;else{k=f-257;m=ga[k];0=a&&(this.a=c,d=this.e(),c=this.a);for(;m--;)d[c]=d[c++-l]}for(;8<=this.d;)this.d-=8,this.c--;this.a=c}; -L.prototype.C=function(e,b){var d=this.b,c=this.a;this.r=e;for(var a=d.length,f,k,l,m;256!==(f=R(this,e));)if(256>f)c>=a&&(d=this.e(),a=d.length),d[c++]=f;else{k=f-257;m=ga[k];0a&&(d=this.e(),a=d.length);for(;m--;)d[c]=d[c++-l]}for(;8<=this.d;)this.d-=8,this.c--;this.a=c}; -L.prototype.e=function(){var e=new (x?Uint8Array:Array)(this.a-32768),b=this.a-32768,d,c,a=this.b;if(x)e.set(a.subarray(32768,e.length));else{d=0;for(c=e.length;dd;++d)a[d]=a[b+d];this.a=32768;return a}; -L.prototype.D=function(e){var b,d=this.input.length/this.c+1|0,c,a,f,k=this.input,l=this.b;e&&("number"===typeof e.t&&(d=e.t),"number"===typeof e.z&&(d+=e.z));2>d?(c=(k.length-this.c)/this.r[2],f=258*(c/2)|0,a=fb&&(this.b.length=b),e=this.b);return this.buffer=e};function $(e){this.input=e;this.c=0;this.m=[];this.s=!1}$.prototype.F=function(){this.s||this.g();return this.m.slice()}; -$.prototype.g=function(){for(var e=this.input.length;this.c>>0;B(a,p,p)!==q&&n(Error("invalid CRC-32 checksum: 0x"+B(a,p,p).toString(16)+" / 0x"+q.toString(16)));b.L= -d=(g[h++]|g[h++]<<8|g[h++]<<16|g[h++]<<24)>>>0;(a.length&4294967295)!==d&&n(Error("invalid input size: "+(a.length&4294967295)+" / "+d));this.m.push(b);this.c=h}this.s=!0;var v=this.m,s,F,H=0,w=0,z;s=0;for(F=v.length;s= 0) { + item._idleTimeoutId = setTimeout(function onTimeout() { + if (item._onTimeout) + item._onTimeout(); + }, msecs); + } +}; + +// That's not how node.js implements it but the exposed api is the same. +exports.setImmediate = typeof setImmediate === "function" ? setImmediate : function(fn) { + var id = nextImmediateId++; + var args = arguments.length < 2 ? false : slice.call(arguments, 1); + + immediateIds[id] = true; + + nextTick(function onNextTick() { + if (immediateIds[id]) { + // fn.call() is faster so we optimize for the common use-case + // @see http://jsperf.com/call-apply-segu + if (args) { + fn.apply(null, args); + } else { + fn.call(null); + } + // Prevent ids from leaking + exports.clearImmediate(id); + } + }); + + return id; +}; + +exports.clearImmediate = typeof clearImmediate === "function" ? clearImmediate : function(id) { + delete immediateIds[id]; +}; +}).call(this)}).call(this,require("timers").setImmediate,require("timers").clearImmediate) +},{"process/browser.js":5,"timers":6}],7:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -6852,11 +9384,11 @@ d=(g[h++]|g[h++]<<8|g[h++]<<16|g[h++]<<24)>>>0;(a.length&4294967295)!==d&&n(Erro "use strict"; -var ViterbiBuilder = require("./viterbi/ViterbiBuilder"); -var ViterbiSearcher = require("./viterbi/ViterbiSearcher"); -var IpadicFormatter = require("./util/IpadicFormatter"); +const ViterbiBuilder = require("./viterbi/ViterbiBuilder"); +const ViterbiSearcher = require("./viterbi/ViterbiSearcher"); +const IpadicFormatter = require("./util/IpadicFormatter"); -var PUNCTUATION = /、|。/; +const PUNCTUATION = /、|。/; /** * Tokenizer @@ -6877,14 +9409,14 @@ function Tokenizer(dic) { * @returns {Array.} Sentences end with punctuation */ Tokenizer.splitByPunctuation = function (input) { - var sentences = []; - var tail = input; - while (true) { + const sentences = []; + let tail = input; + while (true) { if (tail === "") { break; } - var index = tail.search(PUNCTUATION); - if (index < 0) { + const index = tail.search(PUNCTUATION); + if (index < 0) { sentences.push(tail); break; } @@ -6900,11 +9432,11 @@ Tokenizer.splitByPunctuation = function (input) { * @returns {Array} Tokens */ Tokenizer.prototype.tokenize = function (text) { - var sentences = Tokenizer.splitByPunctuation(text); - var tokens = []; - for (var i = 0; i < sentences.length; i++) { - var sentence = sentences[i]; - this.tokenizeForSentence(sentence, tokens); + const sentences = Tokenizer.splitByPunctuation(text); + const tokens = []; + for (let i = 0; i < sentences.length; i++) { + const sentence = sentences[i]; + this.tokenizeForSentence(sentence, tokens); } return tokens; }; @@ -6913,18 +9445,18 @@ Tokenizer.prototype.tokenizeForSentence = function (sentence, tokens) { if (tokens == null) { tokens = []; } - var lattice = this.getLattice(sentence); - var best_path = this.viterbi_searcher.search(lattice); - var last_pos = 0; - if (tokens.length > 0) { + const lattice = this.getLattice(sentence); + const best_path = this.viterbi_searcher.search(lattice); + let last_pos = 0; + if (tokens.length > 0) { last_pos = tokens[tokens.length - 1].word_position; } - for (var j = 0; j < best_path.length; j++) { - var node = best_path[j]; + for (let j = 0; j < best_path.length; j++) { + const node = best_path[j]; - var token, features, features_line; - if (node.type === "KNOWN") { + let token, features, features_line; + if (node.type === "KNOWN") { features_line = this.token_info_dictionary.getFeatures(node.name); if (features_line == null) { features = []; @@ -6963,7 +9495,7 @@ Tokenizer.prototype.getLattice = function (text) { module.exports = Tokenizer; -},{"./util/IpadicFormatter":22,"./viterbi/ViterbiBuilder":24,"./viterbi/ViterbiSearcher":27}],7:[function(require,module,exports){ +},{"./util/IpadicFormatter":23,"./viterbi/ViterbiBuilder":25,"./viterbi/ViterbiSearcher":28}],8:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -6983,8 +9515,8 @@ module.exports = Tokenizer; "use strict"; -var Tokenizer = require("./Tokenizer"); -var DictionaryLoader = require("./loader/NodeDictionaryLoader"); +const Tokenizer = require("./Tokenizer"); +const BrowserDictionaryLoader = require("./loader/BrowserDictionaryLoader"); /** * TokenizerBuilder create Tokenizer instance. @@ -7005,8 +9537,8 @@ function TokenizerBuilder(option) { * @param {TokenizerBuilder~onLoad} callback Callback function */ TokenizerBuilder.prototype.build = function (callback) { - var loader = new DictionaryLoader(this.dic_path); - loader.load(function (err, dic) { + const loader = new BrowserDictionaryLoader(this.dic_path); + loader.load(function (err, dic) { callback(err, new Tokenizer(dic)); }); }; @@ -7020,7 +9552,7 @@ TokenizerBuilder.prototype.build = function (callback) { module.exports = TokenizerBuilder; -},{"./Tokenizer":6,"./loader/NodeDictionaryLoader":19}],8:[function(require,module,exports){ +},{"./Tokenizer":7,"./loader/BrowserDictionaryLoader":20}],9:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7059,7 +9591,7 @@ function CharacterClass(class_id, class_name, is_always_invoke, is_grouping, max module.exports = CharacterClass; -},{}],9:[function(require,module,exports){ +},{}],10:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7079,11 +9611,11 @@ module.exports = CharacterClass; "use strict"; -var InvokeDefinitionMap = require("./InvokeDefinitionMap"); -var CharacterClass = require("./CharacterClass"); -var SurrogateAwareString = require("../util/SurrogateAwareString"); +const InvokeDefinitionMap = require("./InvokeDefinitionMap"); +const CharacterClass = require("./CharacterClass"); +const SurrogateAwareString = require("../util/SurrogateAwareString"); -var DEFAULT_CATEGORY = "DEFAULT"; +const DEFAULT_CATEGORY = "DEFAULT"; /** * CharacterDefinition represents char.def file and @@ -7104,19 +9636,19 @@ function CharacterDefinition() { * @returns {CharacterDefinition} */ CharacterDefinition.load = function (cat_map_buffer, compat_cat_map_buffer, invoke_def_buffer) { - var char_def = new CharacterDefinition(); - char_def.character_category_map = cat_map_buffer; + const char_def = new CharacterDefinition(); + char_def.character_category_map = cat_map_buffer; char_def.compatible_category_map = compat_cat_map_buffer; char_def.invoke_definition_map = InvokeDefinitionMap.load(invoke_def_buffer); return char_def; }; CharacterDefinition.parseCharCategory = function (class_id, parsed_category_def) { - var category = parsed_category_def[1]; - var invoke = parseInt(parsed_category_def[2]); - var grouping = parseInt(parsed_category_def[3]); - var max_length = parseInt(parsed_category_def[4]); - if (!isFinite(invoke) || (invoke !== 0 && invoke !== 1)) { + const category = parsed_category_def[1]; + const invoke = parseInt(parsed_category_def[2]); + const grouping = parseInt(parsed_category_def[3]); + const max_length = parseInt(parsed_category_def[4]); + if (!isFinite(invoke) || (invoke !== 0 && invoke !== 1)) { console.log("char.def parse error. INVOKE is 0 or 1 in:" + invoke); return null; } @@ -7128,28 +9660,28 @@ CharacterDefinition.parseCharCategory = function (class_id, parsed_category_def) console.log("char.def parse error. LENGTH is 1 to n:" + max_length); return null; } - var is_invoke = (invoke === 1); - var is_grouping = (grouping === 1); + const is_invoke = (invoke === 1); + const is_grouping = (grouping === 1); - return new CharacterClass(class_id, category, is_invoke, is_grouping, max_length); + return new CharacterClass(class_id, category, is_invoke, is_grouping, max_length); }; CharacterDefinition.parseCategoryMapping = function (parsed_category_mapping) { - var start = parseInt(parsed_category_mapping[1]); - var default_category = parsed_category_mapping[2]; - var compatible_category = (3 < parsed_category_mapping.length) ? parsed_category_mapping.slice(3) : []; - if (!isFinite(start) || start < 0 || start > 0xFFFF) { + const start = parseInt(parsed_category_mapping[1]); + const default_category = parsed_category_mapping[2]; + const compatible_category = (3 < parsed_category_mapping.length) ? parsed_category_mapping.slice(3) : []; + if (!isFinite(start) || start < 0 || start > 0xFFFF) { console.log("char.def parse error. CODE is invalid:" + start); } return { start: start, default: default_category, compatible: compatible_category}; }; CharacterDefinition.parseRangeCategoryMapping = function (parsed_category_mapping) { - var start = parseInt(parsed_category_mapping[1]); - var end = parseInt(parsed_category_mapping[2]); - var default_category = parsed_category_mapping[3]; - var compatible_category = (4 < parsed_category_mapping.length) ? parsed_category_mapping.slice(4) : []; - if (!isFinite(start) || start < 0 || start > 0xFFFF) { + const start = parseInt(parsed_category_mapping[1]); + const end = parseInt(parsed_category_mapping[2]); + const default_category = parsed_category_mapping[3]; + const compatible_category = (4 < parsed_category_mapping.length) ? parsed_category_mapping.slice(4) : []; + if (!isFinite(start) || start < 0 || start > 0xFFFF) { console.log("char.def parse error. CODE is invalid:" + start); } if (!isFinite(end) || end < 0 || end > 0xFFFF) { @@ -7164,35 +9696,35 @@ CharacterDefinition.parseRangeCategoryMapping = function (parsed_category_mappin */ CharacterDefinition.prototype.initCategoryMappings = function (category_mapping) { // Initialize map by DEFAULT class - var code_point; - if (category_mapping != null) { - for (var i = 0; i < category_mapping.length; i++) { - var mapping = category_mapping[i]; - var end = mapping.end || mapping.start; - for (code_point = mapping.start; code_point <= end; code_point++) { + let code_point; + if (category_mapping != null) { + for (let i = 0; i < category_mapping.length; i++) { + const mapping = category_mapping[i]; + const end = mapping.end || mapping.start; + for (code_point = mapping.start; code_point <= end; code_point++) { // Default Category class ID this.character_category_map[code_point] = this.invoke_definition_map.lookup(mapping.default); - for (var j = 0; j < mapping.compatible.length; j++) { - var bitset = this.compatible_category_map[code_point]; - var compatible_category = mapping.compatible[j]; - if (compatible_category == null) { + for (let j = 0; j < mapping.compatible.length; j++) { + let bitset = this.compatible_category_map[code_point]; + const compatible_category = mapping.compatible[j]; + if (compatible_category == null) { continue; } - var class_id = this.invoke_definition_map.lookup(compatible_category); // Default Category + const class_id = this.invoke_definition_map.lookup(compatible_category); // Default Category if (class_id == null) { continue; } - var class_id_bit = 1 << class_id; - bitset = bitset | class_id_bit; // Set a bit of class ID 例えば、class_idが3のとき、3ビット目に1を立てる + const class_id_bit = 1 << class_id; + bitset = bitset | class_id_bit; // Set a bit of class ID 例えば、class_idが3のとき、3ビット目に1を立てる this.compatible_category_map[code_point] = bitset; } } } } - var default_id = this.invoke_definition_map.lookup(DEFAULT_CATEGORY); - if (default_id == null) { + const default_id = this.invoke_definition_map.lookup(DEFAULT_CATEGORY); + if (default_id == null) { return; } for (code_point = 0; code_point < this.character_category_map.length; code_point++) { @@ -7210,16 +9742,16 @@ CharacterDefinition.prototype.initCategoryMappings = function (category_mapping) * @returns {Array.} character classes */ CharacterDefinition.prototype.lookupCompatibleCategory = function (ch) { - var classes = []; - - /* - if (SurrogateAwareString.isSurrogatePair(ch)) { - // Surrogate pair character codes can not be defined by char.def - return classes; - }*/ - var code = ch.charCodeAt(0); - var integer; - if (code < this.compatible_category_map.length) { + const classes = []; + + /* + if (SurrogateAwareString.isSurrogatePair(ch)) { + // Surrogate pair character codes can not be defined by char.def + return classes; + }*/ + const code = ch.charCodeAt(0); + let integer; + if (code < this.compatible_category_map.length) { integer = this.compatible_category_map[code]; // Bitset } @@ -7227,10 +9759,10 @@ CharacterDefinition.prototype.lookupCompatibleCategory = function (ch) { return classes; } - for (var bit = 0; bit < 32; bit++) { // Treat "bit" as a class ID + for (let bit = 0; bit < 32; bit++) { // Treat "bit" as a class ID if (((integer << (31 - bit)) >>> 31) === 1) { - var character_class = this.invoke_definition_map.getCharacterClass(bit); - if (character_class == null) { + const character_class = this.invoke_definition_map.getCharacterClass(bit); + if (character_class == null) { continue; } classes.push(character_class); @@ -7247,10 +9779,10 @@ CharacterDefinition.prototype.lookupCompatibleCategory = function (ch) { */ CharacterDefinition.prototype.lookup = function (ch) { - var class_id; + let class_id; - var code = ch.charCodeAt(0); - if (SurrogateAwareString.isSurrogatePair(ch)) { + const code = ch.charCodeAt(0); + if (SurrogateAwareString.isSurrogatePair(ch)) { // Surrogate pair character codes can not be defined by char.def, so set DEFAULT category class_id = this.invoke_definition_map.lookup(DEFAULT_CATEGORY); } else if (code < this.character_category_map.length) { @@ -7266,7 +9798,7 @@ CharacterDefinition.prototype.lookup = function (ch) { module.exports = CharacterDefinition; -},{"../util/SurrogateAwareString":23,"./CharacterClass":8,"./InvokeDefinitionMap":12}],10:[function(require,module,exports){ +},{"../util/SurrogateAwareString":24,"./CharacterClass":9,"./InvokeDefinitionMap":13}],11:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7304,16 +9836,16 @@ function ConnectionCosts(forward_dimension, backward_dimension) { } ConnectionCosts.prototype.put = function (forward_id, backward_id, cost) { - var index = forward_id * this.backward_dimension + backward_id + 2; - if (this.buffer.length < index + 1) { + const index = forward_id * this.backward_dimension + backward_id + 2; + if (this.buffer.length < index + 1) { throw "ConnectionCosts buffer overflow"; } this.buffer[index] = cost; }; ConnectionCosts.prototype.get = function (forward_id, backward_id) { - var index = forward_id * this.backward_dimension + backward_id + 2; - if (this.buffer.length < index + 1) { + const index = forward_id * this.backward_dimension + backward_id + 2; + if (this.buffer.length < index + 1) { throw "ConnectionCosts buffer overflow"; } return this.buffer[index]; @@ -7327,7 +9859,7 @@ ConnectionCosts.prototype.loadConnectionCosts = function (connection_costs_buffe module.exports = ConnectionCosts; -},{}],11:[function(require,module,exports){ +},{}],12:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7347,10 +9879,10 @@ module.exports = ConnectionCosts; "use strict"; -var doublearray = require("doublearray"); -var TokenInfoDictionary = require("./TokenInfoDictionary"); -var ConnectionCosts = require("./ConnectionCosts"); -var UnknownDictionary = require("./UnknownDictionary"); +const doublearray = require("doublearray"); +const TokenInfoDictionary = require("./TokenInfoDictionary"); +const ConnectionCosts = require("./ConnectionCosts"); +const UnknownDictionary = require("./UnknownDictionary"); /** * Dictionaries container for Tokenizer @@ -7411,7 +9943,7 @@ DynamicDictionaries.prototype.loadUnknownDictionaries = function (unk_buffer, un module.exports = DynamicDictionaries; -},{"./ConnectionCosts":10,"./TokenInfoDictionary":13,"./UnknownDictionary":14,"doublearray":2}],12:[function(require,module,exports){ +},{"./ConnectionCosts":11,"./TokenInfoDictionary":14,"./UnknownDictionary":15,"doublearray":2}],13:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7431,8 +9963,8 @@ module.exports = DynamicDictionaries; "use strict"; -var ByteBuffer = require("../util/ByteBuffer"); -var CharacterClass = require("./CharacterClass"); +const ByteBuffer = require("../util/ByteBuffer"); +const CharacterClass = require("./CharacterClass"); /** * InvokeDefinitionMap represents invoke definition a part of char.def @@ -7449,17 +9981,17 @@ function InvokeDefinitionMap() { * @returns {InvokeDefinitionMap} */ InvokeDefinitionMap.load = function (invoke_def_buffer) { - var invoke_def = new InvokeDefinitionMap(); - var character_category_definition = []; + const invoke_def = new InvokeDefinitionMap(); + const character_category_definition = []; - var buffer = new ByteBuffer(invoke_def_buffer); - while (buffer.position + 1 < buffer.size()) { - var class_id = character_category_definition.length; - var is_always_invoke = buffer.get(); - var is_grouping = buffer.get(); - var max_length = buffer.getInt(); - var class_name = buffer.getString(); - character_category_definition.push(new CharacterClass(class_id, class_name, is_always_invoke, is_grouping, max_length)); + const buffer = new ByteBuffer(invoke_def_buffer); + while (buffer.position + 1 < buffer.size()) { + const class_id = character_category_definition.length; + const is_always_invoke = buffer.get(); + const is_grouping = buffer.get(); + const max_length = buffer.getInt(); + const class_name = buffer.getString(); + character_category_definition.push(new CharacterClass(class_id, class_name, is_always_invoke, is_grouping, max_length)); } invoke_def.init(character_category_definition); @@ -7475,9 +10007,9 @@ InvokeDefinitionMap.prototype.init = function (character_category_definition) { if (character_category_definition == null) { return; } - for (var i = 0; i < character_category_definition.length; i++) { - var character_class = character_category_definition[i]; - this.map[i] = character_class; + for (let i = 0; i < character_category_definition.length; i++) { + const character_class = character_category_definition[i]; + this.map[i] = character_class; this.lookup_table[character_class.class_name] = i; } }; @@ -7497,8 +10029,8 @@ InvokeDefinitionMap.prototype.getCharacterClass = function (class_id) { * @returns {number} class_id */ InvokeDefinitionMap.prototype.lookup = function (class_name) { - var class_id = this.lookup_table[class_name]; - if (class_id == null) { + const class_id = this.lookup_table[class_name]; + if (class_id == null) { return null; } return class_id; @@ -7509,10 +10041,10 @@ InvokeDefinitionMap.prototype.lookup = function (class_name) { * @returns {Uint8Array} */ InvokeDefinitionMap.prototype.toBuffer = function () { - var buffer = new ByteBuffer(); - for (var i = 0; i < this.map.length; i++) { - var char_class = this.map[i]; - buffer.put(char_class.is_always_invoke); + const buffer = new ByteBuffer(); + for (let i = 0; i < this.map.length; i++) { + const char_class = this.map[i]; + buffer.put(char_class.is_always_invoke); buffer.put(char_class.is_grouping); buffer.putInt(char_class.max_length); buffer.putString(char_class.class_name); @@ -7523,7 +10055,7 @@ InvokeDefinitionMap.prototype.toBuffer = function () { module.exports = InvokeDefinitionMap; -},{"../util/ByteBuffer":21,"./CharacterClass":8}],13:[function(require,module,exports){ +},{"../util/ByteBuffer":22,"./CharacterClass":9}],14:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7543,7 +10075,7 @@ module.exports = InvokeDefinitionMap; "use strict"; -var ByteBuffer = require("../util/ByteBuffer"); +const ByteBuffer = require("../util/ByteBuffer"); /** * TokenInfoDictionary @@ -7558,28 +10090,28 @@ function TokenInfoDictionary() { // left_id right_id word_cost ... // ^ this position is token_info_id TokenInfoDictionary.prototype.buildDictionary = function (entries) { - var dictionary_entries = {}; // using as hashmap, string -> string (word_id -> surface_form) to build dictionary + const dictionary_entries = {}; // using as hashmap, string -> string (word_id -> surface_form) to build dictionary - for (var i = 0; i < entries.length; i++) { - var entry = entries[i]; + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; - if (entry.length < 4) { + if (entry.length < 4) { continue; } - var surface_form = entry[0]; - var left_id = entry[1]; - var right_id = entry[2]; - var word_cost = entry[3]; - var feature = entry.slice(4).join(","); // TODO Optimize + const surface_form = entry[0]; + const left_id = entry[1]; + const right_id = entry[2]; + const word_cost = entry[3]; + const feature = entry.slice(4).join(","); // TODO Optimize // Assertion if (!isFinite(left_id) || !isFinite(right_id) || !isFinite(word_cost)) { console.log(entry); } - var token_info_id = this.put(left_id, right_id, word_cost, surface_form, feature); - dictionary_entries[token_info_id] = surface_form; + const token_info_id = this.put(left_id, right_id, word_cost, surface_form, feature); + dictionary_entries[token_info_id] = surface_form; } // Remove last unused area @@ -7590,10 +10122,10 @@ TokenInfoDictionary.prototype.buildDictionary = function (entries) { }; TokenInfoDictionary.prototype.put = function (left_id, right_id, word_cost, surface_form, feature) { - var token_info_id = this.dictionary.position; - var pos_id = this.pos_buffer.position; + const token_info_id = this.dictionary.position; + const pos_id = this.pos_buffer.position; - this.dictionary.putShort(left_id); + this.dictionary.putShort(left_id); this.dictionary.putShort(right_id); this.dictionary.putShort(word_cost); this.dictionary.putInt(pos_id); @@ -7603,8 +10135,8 @@ TokenInfoDictionary.prototype.put = function (left_id, right_id, word_cost, surf }; TokenInfoDictionary.prototype.addMapping = function (source, target) { - var mapping = this.target_map[source]; - if (mapping == null) { + let mapping = this.target_map[source]; + if (mapping == null) { mapping = []; } mapping.push(target); @@ -7613,15 +10145,15 @@ TokenInfoDictionary.prototype.addMapping = function (source, target) { }; TokenInfoDictionary.prototype.targetMapToBuffer = function () { - var buffer = new ByteBuffer(); - var map_keys_size = Object.keys(this.target_map).length; - buffer.putInt(map_keys_size); - for (var key in this.target_map) { - var values = this.target_map[key]; // Array - var map_values_size = values.length; - buffer.putInt(parseInt(key)); + const buffer = new ByteBuffer(); + const map_keys_size = Object.keys(this.target_map).length; + buffer.putInt(map_keys_size); + for (let key in this.target_map) { + const values = this.target_map[key]; // Array + const map_values_size = values.length; + buffer.putInt(parseInt(key)); buffer.putInt(map_values_size); - for (var i = 0; i < values.length; i++) { + for (let i = 0; i < values.length; i++) { buffer.putInt(values[i]); } } @@ -7642,19 +10174,19 @@ TokenInfoDictionary.prototype.loadPosVector = function (array_buffer) { // from tid_map.dat TokenInfoDictionary.prototype.loadTargetMap = function (array_buffer) { - var buffer = new ByteBuffer(array_buffer); - buffer.position = 0; + const buffer = new ByteBuffer(array_buffer); + buffer.position = 0; this.target_map = {}; buffer.readInt(); // map_keys_size while (true) { if (buffer.buffer.length < buffer.position + 1) { break; } - var key = buffer.readInt(); - var map_values_size = buffer.readInt(); - for (var i = 0; i < map_values_size; i++) { - var value = buffer.readInt(); - this.addMapping(key, value); + const key = buffer.readInt(); + const map_values_size = buffer.readInt(); + for (let i = 0; i < map_values_size; i++) { + const value = buffer.readInt(); + this.addMapping(key, value); } } return this; @@ -7666,18 +10198,18 @@ TokenInfoDictionary.prototype.loadTargetMap = function (array_buffer) { * @returns {string} Features string concatenated by "," */ TokenInfoDictionary.prototype.getFeatures = function (token_info_id_str) { - var token_info_id = parseInt(token_info_id_str); - if (isNaN(token_info_id)) { + const token_info_id = parseInt(token_info_id_str); + if (isNaN(token_info_id)) { // TODO throw error return ""; } - var pos_id = this.dictionary.getInt(token_info_id + 6); - return this.pos_buffer.getString(pos_id); + const pos_id = this.dictionary.getInt(token_info_id + 6); + return this.pos_buffer.getString(pos_id); }; module.exports = TokenInfoDictionary; -},{"../util/ByteBuffer":21}],14:[function(require,module,exports){ +},{"../util/ByteBuffer":22}],15:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7697,9 +10229,9 @@ module.exports = TokenInfoDictionary; "use strict"; -var TokenInfoDictionary = require("./TokenInfoDictionary"); -var CharacterDefinition = require("./CharacterDefinition"); -var ByteBuffer = require("../util/ByteBuffer"); +const TokenInfoDictionary = require("./TokenInfoDictionary"); +const CharacterDefinition = require("./CharacterDefinition"); +const ByteBuffer = require("../util/ByteBuffer"); /** * UnknownDictionary @@ -7737,7 +10269,7 @@ UnknownDictionary.prototype.loadUnknownDictionaries = function (unk_buffer, unk_ module.exports = UnknownDictionary; -},{"../util/ByteBuffer":21,"./CharacterDefinition":9,"./TokenInfoDictionary":13}],15:[function(require,module,exports){ +},{"../util/ByteBuffer":22,"./CharacterDefinition":10,"./TokenInfoDictionary":14}],16:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7757,12 +10289,12 @@ module.exports = UnknownDictionary; "use strict"; -var CharacterDefinition = require("../CharacterDefinition"); -var InvokeDefinitionMap = require("../InvokeDefinitionMap"); +const CharacterDefinition = require("../CharacterDefinition"); +const InvokeDefinitionMap = require("../InvokeDefinitionMap"); -var CATEGORY_DEF_PATTERN = /^(\w+)\s+(\d)\s+(\d)\s+(\d)/; -var CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; -var RANGE_CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})\.\.(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; +const CATEGORY_DEF_PATTERN = /^(\w+)\s+(\d)\s+(\d)\s+(\d)/; +const CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; +const RANGE_CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})\.\.(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; /** * CharacterDefinitionBuilder @@ -7776,25 +10308,25 @@ function CharacterDefinitionBuilder() { } CharacterDefinitionBuilder.prototype.putLine = function (line) { - var parsed_category_def = CATEGORY_DEF_PATTERN.exec(line); - if (parsed_category_def != null) { - var class_id = this.character_category_definition.length; - var char_class = CharacterDefinition.parseCharCategory(class_id, parsed_category_def); - if (char_class == null) { + const parsed_category_def = CATEGORY_DEF_PATTERN.exec(line); + if (parsed_category_def != null) { + const class_id = this.character_category_definition.length; + const char_class = CharacterDefinition.parseCharCategory(class_id, parsed_category_def); + if (char_class == null) { return; } this.character_category_definition.push(char_class); return; } - var parsed_category_mapping = CATEGORY_MAPPING_PATTERN.exec(line); - if (parsed_category_mapping != null) { - var mapping = CharacterDefinition.parseCategoryMapping(parsed_category_mapping); - this.category_mapping.push(mapping); + const parsed_category_mapping = CATEGORY_MAPPING_PATTERN.exec(line); + if (parsed_category_mapping != null) { + const mapping = CharacterDefinition.parseCategoryMapping(parsed_category_mapping); + this.category_mapping.push(mapping); } - var parsed_range_category_mapping = RANGE_CATEGORY_MAPPING_PATTERN.exec(line); - if (parsed_range_category_mapping != null) { - var range_mapping = CharacterDefinition.parseRangeCategoryMapping(parsed_range_category_mapping); - this.category_mapping.push(range_mapping); + const parsed_range_category_mapping = RANGE_CATEGORY_MAPPING_PATTERN.exec(line); + if (parsed_range_category_mapping != null) { + const range_mapping = CharacterDefinition.parseRangeCategoryMapping(parsed_range_category_mapping); + this.category_mapping.push(range_mapping); } }; @@ -7807,7 +10339,7 @@ CharacterDefinitionBuilder.prototype.build = function () { module.exports = CharacterDefinitionBuilder; -},{"../CharacterDefinition":9,"../InvokeDefinitionMap":12}],16:[function(require,module,exports){ +},{"../CharacterDefinition":10,"../InvokeDefinitionMap":13}],17:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7827,7 +10359,7 @@ module.exports = CharacterDefinitionBuilder; "use strict"; -var ConnectionCosts = require("../ConnectionCosts"); +const ConnectionCosts = require("../ConnectionCosts"); /** * Builder class for constructing ConnectionCosts object @@ -7840,11 +10372,11 @@ function ConnectionCostsBuilder() { ConnectionCostsBuilder.prototype.putLine = function (line) { if (this.lines === 0) { - var dimensions = line.split(" "); - var forward_dimension = dimensions[0]; - var backward_dimension = dimensions[1]; + const dimensions = line.split(" "); + const forward_dimension = dimensions[0]; + const backward_dimension = dimensions[1]; - if (forward_dimension < 0 || backward_dimension < 0) { + if (forward_dimension < 0 || backward_dimension < 0) { throw "Parse error of matrix.def"; } @@ -7853,17 +10385,17 @@ ConnectionCostsBuilder.prototype.putLine = function (line) { return this; } - var costs = line.split(" "); + const costs = line.split(" "); - if (costs.length !== 3) { + if (costs.length !== 3) { return this; } - var forward_id = parseInt(costs[0]); - var backward_id = parseInt(costs[1]); - var cost = parseInt(costs[2]); + const forward_id = parseInt(costs[0]); + const backward_id = parseInt(costs[1]); + const cost = parseInt(costs[2]); - if (forward_id < 0 || backward_id < 0 || !isFinite(forward_id) || !isFinite(backward_id) || + if (forward_id < 0 || backward_id < 0 || !isFinite(forward_id) || !isFinite(backward_id) || this.connection_cost.forward_dimension <= forward_id || this.connection_cost.backward_dimension <= backward_id) { throw "Parse error of matrix.def"; } @@ -7879,7 +10411,7 @@ ConnectionCostsBuilder.prototype.build = function () { module.exports = ConnectionCostsBuilder; -},{"../ConnectionCosts":10}],17:[function(require,module,exports){ +},{"../ConnectionCosts":11}],18:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -7899,12 +10431,12 @@ module.exports = ConnectionCostsBuilder; "use strict"; -var doublearray = require("doublearray"); -var DynamicDictionaries = require("../DynamicDictionaries"); -var TokenInfoDictionary = require("../TokenInfoDictionary"); -var ConnectionCostsBuilder = require("./ConnectionCostsBuilder"); -var CharacterDefinitionBuilder = require("./CharacterDefinitionBuilder"); -var UnknownDictionary = require("../UnknownDictionary"); +const doublearray = require("doublearray"); +const DynamicDictionaries = require("../DynamicDictionaries"); +const TokenInfoDictionary = require("../TokenInfoDictionary"); +const ConnectionCostsBuilder = require("./ConnectionCostsBuilder"); +const CharacterDefinitionBuilder = require("./CharacterDefinitionBuilder"); +const UnknownDictionary = require("../UnknownDictionary"); /** * Build dictionaries (token info, connection costs) @@ -7928,8 +10460,8 @@ function DictionaryBuilder() { } DictionaryBuilder.prototype.addTokenInfoDictionary = function (line) { - var new_entry = line.split(","); - this.tid_entries.push(new_entry); + const new_entry = line.split(","); + this.tid_entries.push(new_entry); return this; }; @@ -7957,10 +10489,10 @@ DictionaryBuilder.prototype.putUnkDefLine = function (line) { }; DictionaryBuilder.prototype.build = function () { - var dictionaries = this.buildTokenInfoDictionary(); - var unknown_dictionary = this.buildUnknownDictionary(); + const dictionaries = this.buildTokenInfoDictionary(); + const unknown_dictionary = this.buildUnknownDictionary(); - return new DynamicDictionaries(dictionaries.trie, dictionaries.token_info_dictionary, this.cc_builder.build(), unknown_dictionary); + return new DynamicDictionaries(dictionaries.trie, dictionaries.token_info_dictionary, this.cc_builder.build(), unknown_dictionary); }; /** @@ -7970,18 +10502,18 @@ DictionaryBuilder.prototype.build = function () { */ DictionaryBuilder.prototype.buildTokenInfoDictionary = function () { - var token_info_dictionary = new TokenInfoDictionary(); + const token_info_dictionary = new TokenInfoDictionary(); - // using as hashmap, string -> string (word_id -> surface_form) to build dictionary - var dictionary_entries = token_info_dictionary.buildDictionary(this.tid_entries); + // using as hashmap, string -> string (word_id -> surface_form) to build dictionary + const dictionary_entries = token_info_dictionary.buildDictionary(this.tid_entries); - var trie = this.buildDoubleArray(); + const trie = this.buildDoubleArray(); - for (var token_info_id in dictionary_entries) { - var surface_form = dictionary_entries[token_info_id]; - var trie_id = trie.lookup(surface_form); + for (let token_info_id in dictionary_entries) { + const surface_form = dictionary_entries[token_info_id]; + const trie_id = trie.lookup(surface_form); - // Assertion + // Assertion // if (trie_id < 0) { // console.log("Not Found:" + surface_form); // } @@ -7997,20 +10529,20 @@ DictionaryBuilder.prototype.buildTokenInfoDictionary = function () { DictionaryBuilder.prototype.buildUnknownDictionary = function () { - var unk_dictionary = new UnknownDictionary(); + const unk_dictionary = new UnknownDictionary(); - // using as hashmap, string -> string (word_id -> surface_form) to build dictionary - var dictionary_entries = unk_dictionary.buildDictionary(this.unk_entries); + // using as hashmap, string -> string (word_id -> surface_form) to build dictionary + const dictionary_entries = unk_dictionary.buildDictionary(this.unk_entries); - var char_def = this.cd_builder.build(); // Create CharacterDefinition + const char_def = this.cd_builder.build(); // Create CharacterDefinition unk_dictionary.characterDefinition(char_def); - for (var token_info_id in dictionary_entries) { - var class_name = dictionary_entries[token_info_id]; - var class_id = char_def.invoke_definition_map.lookup(class_name); + for (let token_info_id in dictionary_entries) { + const class_name = dictionary_entries[token_info_id]; + const class_id = char_def.invoke_definition_map.lookup(class_name); - // Assertion + // Assertion // if (trie_id < 0) { // console.log("Not Found:" + surface_form); // } @@ -8027,19 +10559,19 @@ DictionaryBuilder.prototype.buildUnknownDictionary = function () { * @returns {DoubleArray} Double-Array trie */ DictionaryBuilder.prototype.buildDoubleArray = function () { - var trie_id = 0; - var words = this.tid_entries.map(function (entry) { - var surface_form = entry[0]; - return { k: surface_form, v: trie_id++ }; - }); - - var builder = doublearray.builder(1024 * 1024); - return builder.build(words); + let trie_id = 0; + const words = this.tid_entries.map(function (entry) { + const surface_form = entry[0]; + return {k: surface_form, v: trie_id++}; + }); + + const builder = doublearray.builder(1024 * 1024); + return builder.build(words); }; module.exports = DictionaryBuilder; -},{"../DynamicDictionaries":11,"../TokenInfoDictionary":13,"../UnknownDictionary":14,"./CharacterDefinitionBuilder":15,"./ConnectionCostsBuilder":16,"doublearray":2}],18:[function(require,module,exports){ +},{"../DynamicDictionaries":12,"../TokenInfoDictionary":14,"../UnknownDictionary":15,"./CharacterDefinitionBuilder":16,"./ConnectionCostsBuilder":17,"doublearray":2}],19:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8059,22 +10591,22 @@ module.exports = DictionaryBuilder; "use strict"; -var TokenizerBuilder = require("./TokenizerBuilder"); -var DictionaryBuilder = require("./dict/builder/DictionaryBuilder"); +const TokenizerBuilder = require("./TokenizerBuilder"); +const DictionaryBuilder = require("./dict/builder/DictionaryBuilder"); // Public methods -var kuromoji = { - builder: function (option) { - return new TokenizerBuilder(option); - }, - dictionaryBuilder: function () { - return new DictionaryBuilder(); - } +const kuromoji = { + builder: function (option) { + return new TokenizerBuilder(option); + }, + dictionaryBuilder: function () { + return new DictionaryBuilder(); + } }; module.exports = kuromoji; -},{"./TokenizerBuilder":7,"./dict/builder/DictionaryBuilder":17}],19:[function(require,module,exports){ +},{"./TokenizerBuilder":8,"./dict/builder/DictionaryBuilder":18}],20:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8094,8 +10626,8 @@ module.exports = kuromoji; "use strict"; -var zlib = require("zlibjs/bin/gunzip.min.js"); -var DictionaryLoader = require("./DictionaryLoader"); +const fflate = require("fflate"); +const DictionaryLoader = require("./DictionaryLoader"); /** * BrowserDictionaryLoader inherits DictionaryLoader, using jQuery XHR for download @@ -8114,24 +10646,17 @@ BrowserDictionaryLoader.prototype = Object.create(DictionaryLoader.prototype); * @param {BrowserDictionaryLoader~onLoad} callback Callback function */ BrowserDictionaryLoader.prototype.loadArrayBuffer = function (url, callback) { - var xhr = new XMLHttpRequest(); - xhr.open("GET", url, true); - xhr.responseType = "arraybuffer"; - xhr.onload = function () { - if (this.status > 0 && this.status !== 200) { - callback(xhr.statusText, null); - return; - } - var arraybuffer = this.response; - - var gz = new zlib.Zlib.Gunzip(new Uint8Array(arraybuffer)); - var typed_array = gz.decompress(); - callback(null, typed_array.buffer); - }; - xhr.onerror = function (err) { - callback(err, null); - }; - xhr.send(); + fetch(url).then(function (response) { + if (!response.ok) { + callback(response.statusText, null); + } + response.arrayBuffer().then(function (arraybuffer) { + const gz = fflate.gunzipSync(new Uint8Array(arraybuffer)); + callback(null, gz.buffer); + }); + }).catch(function (exception) { + callback(exception, null); + }); }; /** @@ -8143,7 +10668,7 @@ BrowserDictionaryLoader.prototype.loadArrayBuffer = function (url, callback) { module.exports = BrowserDictionaryLoader; -},{"./DictionaryLoader":20,"zlibjs/bin/gunzip.min.js":5}],20:[function(require,module,exports){ +},{"./DictionaryLoader":21,"fflate":3}],21:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8163,9 +10688,8 @@ module.exports = BrowserDictionaryLoader; "use strict"; -var path = require("path"); -var async = require("async"); -var DynamicDictionaries = require("../dict/DynamicDictionaries"); +const async = require("async"); +const DynamicDictionaries = require("../dict/DynamicDictionaries"); /** * DictionaryLoader base constructor @@ -8186,15 +10710,15 @@ DictionaryLoader.prototype.loadArrayBuffer = function (file, callback) { * @param {DictionaryLoader~onLoad} load_callback Callback function called after loaded */ DictionaryLoader.prototype.load = function (load_callback) { - var dic = this.dic; - var dic_path = this.dic_path; - var loadArrayBuffer = this.loadArrayBuffer; + const dic = this.dic; + const dic_path = this.dic_path; + const loadArrayBuffer = this.loadArrayBuffer; - async.parallel([ + async.parallel([ // Trie function (callback) { async.map([ "base.dat.gz", "check.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -8204,17 +10728,17 @@ DictionaryLoader.prototype.load = function (load_callback) { if(err) { return callback(err); } - var base_buffer = new Int32Array(buffers[0]); - var check_buffer = new Int32Array(buffers[1]); + const base_buffer = new Int32Array(buffers[0]); + const check_buffer = new Int32Array(buffers[1]); - dic.loadTrie(base_buffer, check_buffer); + dic.loadTrie(base_buffer, check_buffer); callback(null); }); }, // Token info dictionaries function (callback) { async.map([ "tid.dat.gz", "tid_pos.dat.gz", "tid_map.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -8224,29 +10748,29 @@ DictionaryLoader.prototype.load = function (load_callback) { if(err) { return callback(err); } - var token_info_buffer = new Uint8Array(buffers[0]); - var pos_buffer = new Uint8Array(buffers[1]); - var target_map_buffer = new Uint8Array(buffers[2]); + const token_info_buffer = new Uint8Array(buffers[0]); + const pos_buffer = new Uint8Array(buffers[1]); + const target_map_buffer = new Uint8Array(buffers[2]); - dic.loadTokenInfoDictionaries(token_info_buffer, pos_buffer, target_map_buffer); + dic.loadTokenInfoDictionaries(token_info_buffer, pos_buffer, target_map_buffer); callback(null); }); }, // Connection cost matrix function (callback) { - loadArrayBuffer(path.join(dic_path, "cc.dat.gz"), function (err, buffer) { + loadArrayBuffer(dic_path + "cc.dat.gz", function (err, buffer) { if(err) { return callback(err); } - var cc_buffer = new Int16Array(buffer); - dic.loadConnectionCosts(cc_buffer); + const cc_buffer = new Int16Array(buffer); + dic.loadConnectionCosts(cc_buffer); callback(null); }); }, // Unknown dictionaries function (callback) { async.map([ "unk.dat.gz", "unk_pos.dat.gz", "unk_map.dat.gz", "unk_char.dat.gz", "unk_compat.dat.gz", "unk_invoke.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -8256,14 +10780,14 @@ DictionaryLoader.prototype.load = function (load_callback) { if(err) { return callback(err); } - var unk_buffer = new Uint8Array(buffers[0]); - var unk_pos_buffer = new Uint8Array(buffers[1]); - var unk_map_buffer = new Uint8Array(buffers[2]); - var cat_map_buffer = new Uint8Array(buffers[3]); - var compat_cat_map_buffer = new Uint32Array(buffers[4]); - var invoke_def_buffer = new Uint8Array(buffers[5]); - - dic.loadUnknownDictionaries(unk_buffer, unk_pos_buffer, unk_map_buffer, cat_map_buffer, compat_cat_map_buffer, invoke_def_buffer); + const unk_buffer = new Uint8Array(buffers[0]); + const unk_pos_buffer = new Uint8Array(buffers[1]); + const unk_map_buffer = new Uint8Array(buffers[2]); + const cat_map_buffer = new Uint8Array(buffers[3]); + const compat_cat_map_buffer = new Uint32Array(buffers[4]); + const invoke_def_buffer = new Uint8Array(buffers[5]); + + dic.loadUnknownDictionaries(unk_buffer, unk_pos_buffer, unk_map_buffer, cat_map_buffer, compat_cat_map_buffer, invoke_def_buffer); // dic.loadUnknownDictionaries(char_buffer, unk_buffer); callback(null); }); @@ -8282,7 +10806,7 @@ DictionaryLoader.prototype.load = function (load_callback) { module.exports = DictionaryLoader; -},{"../dict/DynamicDictionaries":11,"async":1,"path":3}],21:[function(require,module,exports){ +},{"../dict/DynamicDictionaries":12,"async":1}],22:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8308,63 +10832,64 @@ module.exports = DictionaryLoader; * @param {String} str UTF-16 string to convert * @return {Uint8Array} Byte sequence encoded by UTF-8 */ -var stringToUtf8Bytes = function (str) { +const stringToUtf8Bytes = function (str) { - // Max size of 1 character is 4 bytes - var bytes = new Uint8Array(str.length * 4); + // Max size of 1 character is 4 bytes + const bytes = new Uint8Array(str.length * 4); - var i = 0, j = 0; + let i = 0; + let j = 0; - while (i < str.length) { - var unicode_code; + while (i < str.length) { + let unicode_code; - var utf16_code = str.charCodeAt(i++); - if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { - // surrogate pair - var upper = utf16_code; // high surrogate - var lower = str.charCodeAt(i++); // low surrogate - - if (lower >= 0xDC00 && lower <= 0xDFFF) { - unicode_code = - (upper - 0xD800) * (1 << 10) + (1 << 16) + - (lower - 0xDC00); - } else { - // malformed surrogate pair - return null; - } - } else { - // not surrogate code - unicode_code = utf16_code; - } + const utf16_code = str.charCodeAt(i++); + if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { + // surrogate pair + const upper = utf16_code; // high surrogate + const lower = str.charCodeAt(i++); // low surrogate + + if (lower >= 0xDC00 && lower <= 0xDFFF) { + unicode_code = + (upper - 0xD800) * (1 << 10) + (1 << 16) + + (lower - 0xDC00); + } else { + // malformed surrogate pair + return null; + } + } else { + // not surrogate code + unicode_code = utf16_code; + } - if (unicode_code < 0x80) { - // 1-byte - bytes[j++] = unicode_code; + if (unicode_code < 0x80) { + // 1-byte + bytes[j++] = unicode_code; - } else if (unicode_code < (1 << 11)) { - // 2-byte - bytes[j++] = (unicode_code >>> 6) | 0xC0; - bytes[j++] = (unicode_code & 0x3F) | 0x80; + } else if (unicode_code < (1 << 11)) { + // 2-byte + bytes[j++] = (unicode_code >>> 6) | 0xC0; + bytes[j++] = (unicode_code & 0x3F) | 0x80; - } else if (unicode_code < (1 << 16)) { - // 3-byte - bytes[j++] = (unicode_code >>> 12) | 0xE0; - bytes[j++] = ((unicode_code >> 6) & 0x3f) | 0x80; - bytes[j++] = (unicode_code & 0x3F) | 0x80; + } else if (unicode_code < (1 << 16)) { + // 3-byte + bytes[j++] = (unicode_code >>> 12) | 0xE0; + bytes[j++] = ((unicode_code >> 6) & 0x3f) | 0x80; + bytes[j++] = (unicode_code & 0x3F) | 0x80; - } else if (unicode_code < (1 << 21)) { - // 4-byte - bytes[j++] = (unicode_code >>> 18) | 0xF0; - bytes[j++] = ((unicode_code >> 12) & 0x3F) | 0x80; - bytes[j++] = ((unicode_code >> 6) & 0x3F) | 0x80; - bytes[j++] = (unicode_code & 0x3F) | 0x80; + } else if (unicode_code < (1 << 21)) { + // 4-byte + bytes[j++] = (unicode_code >>> 18) | 0xF0; + bytes[j++] = ((unicode_code >> 12) & 0x3F) | 0x80; + bytes[j++] = ((unicode_code >> 6) & 0x3F) | 0x80; + bytes[j++] = (unicode_code & 0x3F) | 0x80; - } else { - // malformed UCS4 code - } + } else { + // malformed UCS4 code } + } - return bytes.subarray(0, j); + return bytes.subarray(0, j); }; /** @@ -8373,48 +10898,48 @@ var stringToUtf8Bytes = function (str) { * @param {Array} bytes UTF-8 byte sequence to convert * @return {String} String encoded by UTF-16 */ -var utf8BytesToString = function (bytes) { +const utf8BytesToString = function (bytes) { - var str = ""; - var code, b1, b2, b3, b4, upper, lower; - var i = 0; + let str = ""; + let code, b1, b2, b3, b4, upper, lower; + let i = 0; - while (i < bytes.length) { - - b1 = bytes[i++]; - - if (b1 < 0x80) { - // 1 byte - code = b1; - } else if ((b1 >> 5) === 0x06) { - // 2 bytes - b2 = bytes[i++]; - code = ((b1 & 0x1f) << 6) | (b2 & 0x3f); - } else if ((b1 >> 4) === 0x0e) { - // 3 bytes - b2 = bytes[i++]; - b3 = bytes[i++]; - code = ((b1 & 0x0f) << 12) | ((b2 & 0x3f) << 6) | (b3 & 0x3f); - } else { - // 4 bytes - b2 = bytes[i++]; - b3 = bytes[i++]; - b4 = bytes[i++]; - code = ((b1 & 0x07) << 18) | ((b2 & 0x3f) << 12) | ((b3 & 0x3f) << 6) | (b4 & 0x3f); - } + while (i < bytes.length) { - if (code < 0x10000) { - str += String.fromCharCode(code); - } else { - // surrogate pair - code -= 0x10000; - upper = (0xD800 | (code >> 10)); - lower = (0xDC00 | (code & 0x3FF)); - str += String.fromCharCode(upper, lower); - } + b1 = bytes[i++]; + + if (b1 < 0x80) { + // 1 byte + code = b1; + } else if ((b1 >> 5) === 0x06) { + // 2 bytes + b2 = bytes[i++]; + code = ((b1 & 0x1f) << 6) | (b2 & 0x3f); + } else if ((b1 >> 4) === 0x0e) { + // 3 bytes + b2 = bytes[i++]; + b3 = bytes[i++]; + code = ((b1 & 0x0f) << 12) | ((b2 & 0x3f) << 6) | (b3 & 0x3f); + } else { + // 4 bytes + b2 = bytes[i++]; + b3 = bytes[i++]; + b4 = bytes[i++]; + code = ((b1 & 0x07) << 18) | ((b2 & 0x3f) << 12) | ((b3 & 0x3f) << 6) | (b4 & 0x3f); + } + + if (code < 0x10000) { + str += String.fromCharCode(code); + } else { + // surrogate pair + code -= 0x10000; + upper = (0xD800 | (code >> 10)); + lower = (0xDC00 | (code & 0x3FF)); + str += String.fromCharCode(upper, lower); } + } - return str; + return str; }; /** @@ -8423,8 +10948,8 @@ var utf8BytesToString = function (bytes) { * @constructor */ function ByteBuffer(arg) { - var initial_size; - if (arg == null) { + let initial_size; + if (arg == null) { initial_size = 1024 * 1024; } else if (typeof arg === "number") { initial_size = arg; @@ -8446,8 +10971,8 @@ ByteBuffer.prototype.size = function () { }; ByteBuffer.prototype.reallocate = function () { - var new_array = new Uint8Array(this.buffer.length * 2); - new_array.set(this.buffer); + const new_array = new Uint8Array(this.buffer.length * 2); + new_array.set(this.buffer); this.buffer = new_array; }; @@ -8479,9 +11004,9 @@ ByteBuffer.prototype.putShort = function (num) { if (0xFFFF < num) { throw num + " is over short value"; } - var lower = (0x00FF & num); - var upper = (0xFF00 & num) >> 8; - this.put(lower); + const lower = (0x00FF & num); + const upper = (0xFF00 & num) >> 8; + this.put(lower); this.put(upper); }; @@ -8494,10 +11019,10 @@ ByteBuffer.prototype.getShort = function (index) { if (this.buffer.length < index + 2) { return 0; } - var lower = this.buffer[index]; - var upper = this.buffer[index + 1]; - var value = (upper << 8) + lower; - if (value & 0x8000) { + const lower = this.buffer[index]; + const upper = this.buffer[index + 1]; + let value = (upper << 8) + lower; + if (value & 0x8000) { value = -((value - 1) ^ 0xFFFF); } return value; @@ -8508,11 +11033,11 @@ ByteBuffer.prototype.putInt = function (num) { if (0xFFFFFFFF < num) { throw num + " is over integer value"; } - var b0 = (0x000000FF & num); - var b1 = (0x0000FF00 & num) >> 8; - var b2 = (0x00FF0000 & num) >> 16; - var b3 = (0xFF000000 & num) >> 24; - this.put(b0); + const b0 = (0x000000FF & num); + const b1 = (0x0000FF00 & num) >> 8; + const b2 = (0x00FF0000 & num) >> 16; + const b3 = (0xFF000000 & num) >> 24; + this.put(b0); this.put(b1); this.put(b2); this.put(b3); @@ -8527,23 +11052,23 @@ ByteBuffer.prototype.getInt = function (index) { if (this.buffer.length < index + 4) { return 0; } - var b0 = this.buffer[index]; - var b1 = this.buffer[index + 1]; - var b2 = this.buffer[index + 2]; - var b3 = this.buffer[index + 3]; + const b0 = this.buffer[index]; + const b1 = this.buffer[index + 1]; + const b2 = this.buffer[index + 2]; + const b3 = this.buffer[index + 3]; - return (b3 << 24) + (b2 << 16) + (b1 << 8) + b0; + return (b3 << 24) + (b2 << 16) + (b1 << 8) + b0; }; ByteBuffer.prototype.readInt = function () { - var pos = this.position; - this.position += 4; + const pos = this.position; + this.position += 4; return this.getInt(pos); }; ByteBuffer.prototype.putString = function (str) { - var bytes = stringToUtf8Bytes(str); - for (var i = 0; i < bytes.length; i++) { + const bytes = stringToUtf8Bytes(str); + for (let i = 0; i < bytes.length; i++) { this.put(bytes[i]); } // put null character as terminal character @@ -8551,9 +11076,9 @@ ByteBuffer.prototype.putString = function (str) { }; ByteBuffer.prototype.getString = function (index) { - var buf = [], - ch; - if (index == null) { + const buf = []; + let ch; + if (index == null) { index = this.position; } while (true) { @@ -8573,7 +11098,7 @@ ByteBuffer.prototype.getString = function (index) { module.exports = ByteBuffer; -},{}],22:[function(require,module,exports){ +},{}],23:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8601,8 +11126,8 @@ function IpadicFormatter() { } IpadicFormatter.prototype.formatEntry = function (word_id, position, type, features) { - var token = {}; - token.word_id = word_id; + const token = {}; + token.word_id = word_id; token.word_type = type; token.word_position = position; @@ -8621,8 +11146,8 @@ IpadicFormatter.prototype.formatEntry = function (word_id, position, type, featu }; IpadicFormatter.prototype.formatUnknownEntry = function (word_id, position, type, features, surface_form) { - var token = {}; - token.word_id = word_id; + const token = {}; + token.word_id = word_id; token.word_type = type; token.word_position = position; @@ -8642,7 +11167,7 @@ IpadicFormatter.prototype.formatUnknownEntry = function (word_id, position, type module.exports = IpadicFormatter; -},{}],23:[function(require,module,exports){ +},{}],24:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8671,9 +11196,9 @@ function SurrogateAwareString(str) { this.str = str; this.index_mapping = []; - for (var pos = 0; pos < str.length; pos++) { - var ch = str.charAt(pos); - this.index_mapping.push(pos); + for (let pos = 0; pos < str.length; pos++) { + const ch = str.charAt(pos); + this.index_mapping.push(pos); if (SurrogateAwareString.isSurrogatePair(ch)) { pos++; } @@ -8686,18 +11211,18 @@ SurrogateAwareString.prototype.slice = function (index) { if (this.index_mapping.length <= index) { return ""; } - var surrogate_aware_index = this.index_mapping[index]; - return this.str.slice(surrogate_aware_index); + const surrogate_aware_index = this.index_mapping[index]; + return this.str.slice(surrogate_aware_index); }; SurrogateAwareString.prototype.charAt = function (index) { if (this.str.length <= index) { return ""; } - var surrogate_aware_start_index = this.index_mapping[index]; - var surrogate_aware_end_index = this.index_mapping[index + 1]; + const surrogate_aware_start_index = this.index_mapping[index]; + const surrogate_aware_end_index = this.index_mapping[index + 1]; - if (surrogate_aware_end_index == null) { + if (surrogate_aware_end_index == null) { return this.str.slice(surrogate_aware_start_index); } return this.str.slice(surrogate_aware_start_index, surrogate_aware_end_index); @@ -8707,10 +11232,10 @@ SurrogateAwareString.prototype.charCodeAt = function (index) { if (this.index_mapping.length <= index) { return NaN; } - var surrogate_aware_index = this.index_mapping[index]; - var upper = this.str.charCodeAt(surrogate_aware_index); - var lower; - if (upper >= 0xD800 && upper <= 0xDBFF && surrogate_aware_index < this.str.length) { + const surrogate_aware_index = this.index_mapping[index]; + const upper = this.str.charCodeAt(surrogate_aware_index); + let lower; + if (upper >= 0xD800 && upper <= 0xDBFF && surrogate_aware_index < this.str.length) { lower = this.str.charCodeAt(surrogate_aware_index + 1); if (lower >= 0xDC00 && lower <= 0xDFFF) { return (upper - 0xD800) * 0x400 + lower - 0xDC00 + 0x10000; @@ -8724,8 +11249,8 @@ SurrogateAwareString.prototype.toString = function () { }; SurrogateAwareString.isSurrogatePair = function (ch) { - var utf16_code = ch.charCodeAt(0); - if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { + const utf16_code = ch.charCodeAt(0); + if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { // surrogate pair return true; } else { @@ -8735,7 +11260,7 @@ SurrogateAwareString.isSurrogatePair = function (ch) { module.exports = SurrogateAwareString; -},{}],24:[function(require,module,exports){ +},{}],25:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8755,9 +11280,9 @@ module.exports = SurrogateAwareString; "use strict"; -var ViterbiNode = require("./ViterbiNode"); -var ViterbiLattice = require("./ViterbiLattice"); -var SurrogateAwareString = require("../util/SurrogateAwareString"); +const ViterbiNode = require("./ViterbiNode"); +const ViterbiLattice = require("./ViterbiLattice"); +const SurrogateAwareString = require("../util/SurrogateAwareString"); /** * ViterbiBuilder builds word lattice (ViterbiLattice) @@ -8776,22 +11301,22 @@ function ViterbiBuilder(dic) { * @returns {ViterbiLattice} Word lattice */ ViterbiBuilder.prototype.build = function (sentence_str) { - var lattice = new ViterbiLattice(); - var sentence = new SurrogateAwareString(sentence_str); - - var key, trie_id, left_id, right_id, word_cost; - for (var pos = 0; pos < sentence.length; pos++) { - var tail = sentence.slice(pos); - var vocabulary = this.trie.commonPrefixSearch(tail); - for (var n = 0; n < vocabulary.length; n++) { // Words in dictionary do not have surrogate pair (only UCS2 set) + const lattice = new ViterbiLattice(); + const sentence = new SurrogateAwareString(sentence_str); + + let key, trie_id, left_id, right_id, word_cost; + for (let pos = 0; pos < sentence.length; pos++) { + const tail = sentence.slice(pos); + const vocabulary = this.trie.commonPrefixSearch(tail); + for (let n = 0; n < vocabulary.length; n++) { // Words in dictionary do not have surrogate pair (only UCS2 set) trie_id = vocabulary[n].v; key = vocabulary[n].k; - var token_info_ids = this.token_info_dictionary.target_map[trie_id]; - for (var i = 0; i < token_info_ids.length; i++) { - var token_info_id = parseInt(token_info_ids[i]); + const token_info_ids = this.token_info_dictionary.target_map[trie_id]; + for (let i = 0; i < token_info_ids.length; i++) { + const token_info_id = parseInt(token_info_ids[i]); - left_id = this.token_info_dictionary.dictionary.getShort(token_info_id); + left_id = this.token_info_dictionary.dictionary.getShort(token_info_id); right_id = this.token_info_dictionary.dictionary.getShort(token_info_id + 2); word_cost = this.token_info_dictionary.dictionary.getShort(token_info_id + 4); @@ -8801,28 +11326,28 @@ ViterbiBuilder.prototype.build = function (sentence_str) { } // Unknown word processing - var surrogate_aware_tail = new SurrogateAwareString(tail); - var head_char = new SurrogateAwareString(surrogate_aware_tail.charAt(0)); - var head_char_class = this.unknown_dictionary.lookup(head_char.toString()); - if (vocabulary == null || vocabulary.length === 0 || head_char_class.is_always_invoke === 1) { + const surrogate_aware_tail = new SurrogateAwareString(tail); + const head_char = new SurrogateAwareString(surrogate_aware_tail.charAt(0)); + const head_char_class = this.unknown_dictionary.lookup(head_char.toString()); + if (vocabulary == null || vocabulary.length === 0 || head_char_class.is_always_invoke === 1) { // Process unknown word key = head_char; if (head_char_class.is_grouping === 1 && 1 < surrogate_aware_tail.length) { - for (var k = 1; k < surrogate_aware_tail.length; k++) { - var next_char = surrogate_aware_tail.charAt(k); - var next_char_class = this.unknown_dictionary.lookup(next_char); - if (head_char_class.class_name !== next_char_class.class_name) { + for (let k = 1; k < surrogate_aware_tail.length; k++) { + const next_char = surrogate_aware_tail.charAt(k); + const next_char_class = this.unknown_dictionary.lookup(next_char); + if (head_char_class.class_name !== next_char_class.class_name) { break; } key += next_char; } } - var unk_ids = this.unknown_dictionary.target_map[head_char_class.class_id]; - for (var j = 0; j < unk_ids.length; j++) { - var unk_id = parseInt(unk_ids[j]); + const unk_ids = this.unknown_dictionary.target_map[head_char_class.class_id]; + for (let j = 0; j < unk_ids.length; j++) { + const unk_id = parseInt(unk_ids[j]); - left_id = this.unknown_dictionary.dictionary.getShort(unk_id); + left_id = this.unknown_dictionary.dictionary.getShort(unk_id); right_id = this.unknown_dictionary.dictionary.getShort(unk_id + 2); word_cost = this.unknown_dictionary.dictionary.getShort(unk_id + 4); @@ -8838,7 +11363,7 @@ ViterbiBuilder.prototype.build = function (sentence_str) { module.exports = ViterbiBuilder; -},{"../util/SurrogateAwareString":23,"./ViterbiLattice":25,"./ViterbiNode":26}],25:[function(require,module,exports){ +},{"../util/SurrogateAwareString":24,"./ViterbiLattice":26,"./ViterbiNode":27}],26:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8858,7 +11383,7 @@ module.exports = ViterbiBuilder; "use strict"; -var ViterbiNode = require("./ViterbiNode"); +const ViterbiNode = require("./ViterbiNode"); /** * ViterbiLattice is a lattice in Viterbi algorithm @@ -8875,13 +11400,13 @@ function ViterbiLattice() { * @param {ViterbiNode} node */ ViterbiLattice.prototype.append = function (node) { - var last_pos = node.start_pos + node.length - 1; - if (this.eos_pos < last_pos) { + const last_pos = node.start_pos + node.length - 1; + if (this.eos_pos < last_pos) { this.eos_pos = last_pos; } - var prev_nodes = this.nodes_end_at[last_pos]; - if (prev_nodes == null) { + let prev_nodes = this.nodes_end_at[last_pos]; + if (prev_nodes == null) { prev_nodes = []; } prev_nodes.push(node); @@ -8893,14 +11418,14 @@ ViterbiLattice.prototype.append = function (node) { * Set ends with EOS (End of Statement) */ ViterbiLattice.prototype.appendEos = function () { - var last_index = this.nodes_end_at.length; - this.eos_pos++; + const last_index = this.nodes_end_at.length; + this.eos_pos++; this.nodes_end_at[last_index] = [ new ViterbiNode(-1, 0, this.eos_pos, 0, "EOS", 0, 0, "") ]; }; module.exports = ViterbiLattice; -},{"./ViterbiNode":26}],26:[function(require,module,exports){ +},{"./ViterbiNode":27}],27:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8951,7 +11476,7 @@ function ViterbiNode(node_name, node_cost, start_pos, length, type, left_id, rig module.exports = ViterbiNode; -},{}],27:[function(require,module,exports){ +},{}],28:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8991,27 +11516,27 @@ ViterbiSearcher.prototype.search = function (lattice) { }; ViterbiSearcher.prototype.forward = function (lattice) { - var i, j, k; - for (i = 1; i <= lattice.eos_pos; i++) { - var nodes = lattice.nodes_end_at[i]; - if (nodes == null) { + let i, j, k; + for (i = 1; i <= lattice.eos_pos; i++) { + const nodes = lattice.nodes_end_at[i]; + if (nodes == null) { continue; } for (j = 0; j < nodes.length; j++) { - var node = nodes[j]; - var cost = Number.MAX_VALUE; - var shortest_prev_node; + const node = nodes[j]; + let cost = Number.MAX_VALUE; + let shortest_prev_node; - var prev_nodes = lattice.nodes_end_at[node.start_pos - 1]; - if (prev_nodes == null) { + const prev_nodes = lattice.nodes_end_at[node.start_pos - 1]; + if (prev_nodes == null) { // TODO process unknown words (repair word lattice) continue; } for (k = 0; k < prev_nodes.length; k++) { - var prev_node = prev_nodes[k]; + const prev_node = prev_nodes[k]; - var edge_cost; - if (node.left_id == null || prev_node.right_id == null) { + let edge_cost; + if (node.left_id == null || prev_node.right_id == null) { // TODO assert console.log("Left or right is null"); edge_cost = 0; @@ -9019,8 +11544,8 @@ ViterbiSearcher.prototype.forward = function (lattice) { edge_cost = this.connection_costs.get(prev_node.right_id, node.left_id); } - var _cost = prev_node.shortest_cost + edge_cost + node.cost; - if (_cost < cost) { + const _cost = prev_node.shortest_cost + edge_cost + node.cost; + if (_cost < cost) { shortest_prev_node = prev_node; cost = _cost; } @@ -9034,11 +11559,11 @@ ViterbiSearcher.prototype.forward = function (lattice) { }; ViterbiSearcher.prototype.backward = function (lattice) { - var shortest_path = []; - var eos = lattice.nodes_end_at[lattice.nodes_end_at.length - 1][0]; + const shortest_path = []; + const eos = lattice.nodes_end_at[lattice.nodes_end_at.length - 1][0]; - var node_back = eos.prev; - if (node_back == null) { + let node_back = eos.prev; + if (node_back == null) { return []; } while (node_back.type !== "BOS") { @@ -9055,5 +11580,5 @@ ViterbiSearcher.prototype.backward = function (lattice) { module.exports = ViterbiSearcher; -},{}]},{},[18])(18) +},{}]},{},[19])(19) }); diff --git a/package.json b/package.json index 6e2d399607e6154a107113b817012694fc1143af..52972980b1ef237244eecc80f12b50774459883c 100644 --- a/package.json +++ b/package.json @@ -9,10 +9,13 @@ "bugs": { "url": "https://github.com/takuyaa/kuromoji.js/issues" }, + "overrides": { + "graceful-fs": "^4.2.3" + }, "dependencies": { - "async": "^2.0.1", + "async": "^2.3.0", "doublearray": "0.0.2", - "zlibjs": "^0.3.1" + "fflate": "0.8.2" }, "devDependencies": { "browserify": "^16.1.1", diff --git a/src/Tokenizer.js b/src/Tokenizer.js index 6c3da9b76a9ea7abada97c0ca63618b449f4661e..75fc74807ba66af70b8de9cbfa2b21c4a5e034b5 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -17,11 +17,11 @@ "use strict"; -var ViterbiBuilder = require("./viterbi/ViterbiBuilder"); -var ViterbiSearcher = require("./viterbi/ViterbiSearcher"); -var IpadicFormatter = require("./util/IpadicFormatter"); +const ViterbiBuilder = require("./viterbi/ViterbiBuilder"); +const ViterbiSearcher = require("./viterbi/ViterbiSearcher"); +const IpadicFormatter = require("./util/IpadicFormatter"); -var PUNCTUATION = /、|。/; +const PUNCTUATION = /、|。/; /** * Tokenizer @@ -42,14 +42,14 @@ function Tokenizer(dic) { * @returns {Array.} Sentences end with punctuation */ Tokenizer.splitByPunctuation = function (input) { - var sentences = []; - var tail = input; - while (true) { + const sentences = []; + let tail = input; + while (true) { if (tail === "") { break; } - var index = tail.search(PUNCTUATION); - if (index < 0) { + const index = tail.search(PUNCTUATION); + if (index < 0) { sentences.push(tail); break; } @@ -65,11 +65,11 @@ Tokenizer.splitByPunctuation = function (input) { * @returns {Array} Tokens */ Tokenizer.prototype.tokenize = function (text) { - var sentences = Tokenizer.splitByPunctuation(text); - var tokens = []; - for (var i = 0; i < sentences.length; i++) { - var sentence = sentences[i]; - this.tokenizeForSentence(sentence, tokens); + const sentences = Tokenizer.splitByPunctuation(text); + const tokens = []; + for (let i = 0; i < sentences.length; i++) { + const sentence = sentences[i]; + this.tokenizeForSentence(sentence, tokens); } return tokens; }; @@ -78,18 +78,18 @@ Tokenizer.prototype.tokenizeForSentence = function (sentence, tokens) { if (tokens == null) { tokens = []; } - var lattice = this.getLattice(sentence); - var best_path = this.viterbi_searcher.search(lattice); - var last_pos = 0; - if (tokens.length > 0) { + const lattice = this.getLattice(sentence); + const best_path = this.viterbi_searcher.search(lattice); + let last_pos = 0; + if (tokens.length > 0) { last_pos = tokens[tokens.length - 1].word_position; } - for (var j = 0; j < best_path.length; j++) { - var node = best_path[j]; + for (let j = 0; j < best_path.length; j++) { + const node = best_path[j]; - var token, features, features_line; - if (node.type === "KNOWN") { + let token, features, features_line; + if (node.type === "KNOWN") { features_line = this.token_info_dictionary.getFeatures(node.name); if (features_line == null) { features = []; diff --git a/src/TokenizerBuilder.js b/src/TokenizerBuilder.js index 9ef5c6a2efc63e8b12735a8a9f1cb08d6c52c20c..223e0c6d5856912b59fba59b255dfae63f7b0d54 100644 --- a/src/TokenizerBuilder.js +++ b/src/TokenizerBuilder.js @@ -17,8 +17,8 @@ "use strict"; -var Tokenizer = require("./Tokenizer"); -var DictionaryLoader = require("./loader/NodeDictionaryLoader"); +const Tokenizer = require("./Tokenizer"); +const BrowserDictionaryLoader = require("./loader/BrowserDictionaryLoader"); /** * TokenizerBuilder create Tokenizer instance. @@ -39,8 +39,8 @@ function TokenizerBuilder(option) { * @param {TokenizerBuilder~onLoad} callback Callback function */ TokenizerBuilder.prototype.build = function (callback) { - var loader = new DictionaryLoader(this.dic_path); - loader.load(function (err, dic) { + const loader = new BrowserDictionaryLoader(this.dic_path); + loader.load(function (err, dic) { callback(err, new Tokenizer(dic)); }); }; diff --git a/src/dict/CharacterDefinition.js b/src/dict/CharacterDefinition.js index 11bb531e3dc1ec686d1f59564c1defb69fcfe974..b728e4da64e1de0b79047b8bdbb67724be8f1067 100644 --- a/src/dict/CharacterDefinition.js +++ b/src/dict/CharacterDefinition.js @@ -17,11 +17,11 @@ "use strict"; -var InvokeDefinitionMap = require("./InvokeDefinitionMap"); -var CharacterClass = require("./CharacterClass"); -var SurrogateAwareString = require("../util/SurrogateAwareString"); +const InvokeDefinitionMap = require("./InvokeDefinitionMap"); +const CharacterClass = require("./CharacterClass"); +const SurrogateAwareString = require("../util/SurrogateAwareString"); -var DEFAULT_CATEGORY = "DEFAULT"; +const DEFAULT_CATEGORY = "DEFAULT"; /** * CharacterDefinition represents char.def file and @@ -42,19 +42,19 @@ function CharacterDefinition() { * @returns {CharacterDefinition} */ CharacterDefinition.load = function (cat_map_buffer, compat_cat_map_buffer, invoke_def_buffer) { - var char_def = new CharacterDefinition(); - char_def.character_category_map = cat_map_buffer; + const char_def = new CharacterDefinition(); + char_def.character_category_map = cat_map_buffer; char_def.compatible_category_map = compat_cat_map_buffer; char_def.invoke_definition_map = InvokeDefinitionMap.load(invoke_def_buffer); return char_def; }; CharacterDefinition.parseCharCategory = function (class_id, parsed_category_def) { - var category = parsed_category_def[1]; - var invoke = parseInt(parsed_category_def[2]); - var grouping = parseInt(parsed_category_def[3]); - var max_length = parseInt(parsed_category_def[4]); - if (!isFinite(invoke) || (invoke !== 0 && invoke !== 1)) { + const category = parsed_category_def[1]; + const invoke = parseInt(parsed_category_def[2]); + const grouping = parseInt(parsed_category_def[3]); + const max_length = parseInt(parsed_category_def[4]); + if (!isFinite(invoke) || (invoke !== 0 && invoke !== 1)) { console.log("char.def parse error. INVOKE is 0 or 1 in:" + invoke); return null; } @@ -66,28 +66,28 @@ CharacterDefinition.parseCharCategory = function (class_id, parsed_category_def) console.log("char.def parse error. LENGTH is 1 to n:" + max_length); return null; } - var is_invoke = (invoke === 1); - var is_grouping = (grouping === 1); + const is_invoke = (invoke === 1); + const is_grouping = (grouping === 1); - return new CharacterClass(class_id, category, is_invoke, is_grouping, max_length); + return new CharacterClass(class_id, category, is_invoke, is_grouping, max_length); }; CharacterDefinition.parseCategoryMapping = function (parsed_category_mapping) { - var start = parseInt(parsed_category_mapping[1]); - var default_category = parsed_category_mapping[2]; - var compatible_category = (3 < parsed_category_mapping.length) ? parsed_category_mapping.slice(3) : []; - if (!isFinite(start) || start < 0 || start > 0xFFFF) { + const start = parseInt(parsed_category_mapping[1]); + const default_category = parsed_category_mapping[2]; + const compatible_category = (3 < parsed_category_mapping.length) ? parsed_category_mapping.slice(3) : []; + if (!isFinite(start) || start < 0 || start > 0xFFFF) { console.log("char.def parse error. CODE is invalid:" + start); } return { start: start, default: default_category, compatible: compatible_category}; }; CharacterDefinition.parseRangeCategoryMapping = function (parsed_category_mapping) { - var start = parseInt(parsed_category_mapping[1]); - var end = parseInt(parsed_category_mapping[2]); - var default_category = parsed_category_mapping[3]; - var compatible_category = (4 < parsed_category_mapping.length) ? parsed_category_mapping.slice(4) : []; - if (!isFinite(start) || start < 0 || start > 0xFFFF) { + const start = parseInt(parsed_category_mapping[1]); + const end = parseInt(parsed_category_mapping[2]); + const default_category = parsed_category_mapping[3]; + const compatible_category = (4 < parsed_category_mapping.length) ? parsed_category_mapping.slice(4) : []; + if (!isFinite(start) || start < 0 || start > 0xFFFF) { console.log("char.def parse error. CODE is invalid:" + start); } if (!isFinite(end) || end < 0 || end > 0xFFFF) { @@ -102,35 +102,35 @@ CharacterDefinition.parseRangeCategoryMapping = function (parsed_category_mappin */ CharacterDefinition.prototype.initCategoryMappings = function (category_mapping) { // Initialize map by DEFAULT class - var code_point; - if (category_mapping != null) { - for (var i = 0; i < category_mapping.length; i++) { - var mapping = category_mapping[i]; - var end = mapping.end || mapping.start; - for (code_point = mapping.start; code_point <= end; code_point++) { + let code_point; + if (category_mapping != null) { + for (let i = 0; i < category_mapping.length; i++) { + const mapping = category_mapping[i]; + const end = mapping.end || mapping.start; + for (code_point = mapping.start; code_point <= end; code_point++) { // Default Category class ID this.character_category_map[code_point] = this.invoke_definition_map.lookup(mapping.default); - for (var j = 0; j < mapping.compatible.length; j++) { - var bitset = this.compatible_category_map[code_point]; - var compatible_category = mapping.compatible[j]; - if (compatible_category == null) { + for (let j = 0; j < mapping.compatible.length; j++) { + let bitset = this.compatible_category_map[code_point]; + const compatible_category = mapping.compatible[j]; + if (compatible_category == null) { continue; } - var class_id = this.invoke_definition_map.lookup(compatible_category); // Default Category + const class_id = this.invoke_definition_map.lookup(compatible_category); // Default Category if (class_id == null) { continue; } - var class_id_bit = 1 << class_id; - bitset = bitset | class_id_bit; // Set a bit of class ID 例えば、class_idが3のとき、3ビット目に1を立てる + const class_id_bit = 1 << class_id; + bitset = bitset | class_id_bit; // Set a bit of class ID 例えば、class_idが3のとき、3ビット目に1を立てる this.compatible_category_map[code_point] = bitset; } } } } - var default_id = this.invoke_definition_map.lookup(DEFAULT_CATEGORY); - if (default_id == null) { + const default_id = this.invoke_definition_map.lookup(DEFAULT_CATEGORY); + if (default_id == null) { return; } for (code_point = 0; code_point < this.character_category_map.length; code_point++) { @@ -148,16 +148,16 @@ CharacterDefinition.prototype.initCategoryMappings = function (category_mapping) * @returns {Array.} character classes */ CharacterDefinition.prototype.lookupCompatibleCategory = function (ch) { - var classes = []; - - /* - if (SurrogateAwareString.isSurrogatePair(ch)) { - // Surrogate pair character codes can not be defined by char.def - return classes; - }*/ - var code = ch.charCodeAt(0); - var integer; - if (code < this.compatible_category_map.length) { + const classes = []; + + /* + if (SurrogateAwareString.isSurrogatePair(ch)) { + // Surrogate pair character codes can not be defined by char.def + return classes; + }*/ + const code = ch.charCodeAt(0); + let integer; + if (code < this.compatible_category_map.length) { integer = this.compatible_category_map[code]; // Bitset } @@ -165,10 +165,10 @@ CharacterDefinition.prototype.lookupCompatibleCategory = function (ch) { return classes; } - for (var bit = 0; bit < 32; bit++) { // Treat "bit" as a class ID + for (let bit = 0; bit < 32; bit++) { // Treat "bit" as a class ID if (((integer << (31 - bit)) >>> 31) === 1) { - var character_class = this.invoke_definition_map.getCharacterClass(bit); - if (character_class == null) { + const character_class = this.invoke_definition_map.getCharacterClass(bit); + if (character_class == null) { continue; } classes.push(character_class); @@ -185,10 +185,10 @@ CharacterDefinition.prototype.lookupCompatibleCategory = function (ch) { */ CharacterDefinition.prototype.lookup = function (ch) { - var class_id; + let class_id; - var code = ch.charCodeAt(0); - if (SurrogateAwareString.isSurrogatePair(ch)) { + const code = ch.charCodeAt(0); + if (SurrogateAwareString.isSurrogatePair(ch)) { // Surrogate pair character codes can not be defined by char.def, so set DEFAULT category class_id = this.invoke_definition_map.lookup(DEFAULT_CATEGORY); } else if (code < this.character_category_map.length) { diff --git a/src/dict/ConnectionCosts.js b/src/dict/ConnectionCosts.js index ba69170ab5a5481ed64c1a585f166c34b302e643..1192b71c419540dbf75353947d36ed00084ae309 100644 --- a/src/dict/ConnectionCosts.js +++ b/src/dict/ConnectionCosts.js @@ -35,16 +35,16 @@ function ConnectionCosts(forward_dimension, backward_dimension) { } ConnectionCosts.prototype.put = function (forward_id, backward_id, cost) { - var index = forward_id * this.backward_dimension + backward_id + 2; - if (this.buffer.length < index + 1) { + const index = forward_id * this.backward_dimension + backward_id + 2; + if (this.buffer.length < index + 1) { throw "ConnectionCosts buffer overflow"; } this.buffer[index] = cost; }; ConnectionCosts.prototype.get = function (forward_id, backward_id) { - var index = forward_id * this.backward_dimension + backward_id + 2; - if (this.buffer.length < index + 1) { + const index = forward_id * this.backward_dimension + backward_id + 2; + if (this.buffer.length < index + 1) { throw "ConnectionCosts buffer overflow"; } return this.buffer[index]; diff --git a/src/dict/DynamicDictionaries.js b/src/dict/DynamicDictionaries.js index 452955aa375f54dec5192f73915f5fe74f4b07dc..2b4006a33c751cbb9885ca7f34afaa01c88535f2 100644 --- a/src/dict/DynamicDictionaries.js +++ b/src/dict/DynamicDictionaries.js @@ -17,10 +17,10 @@ "use strict"; -var doublearray = require("doublearray"); -var TokenInfoDictionary = require("./TokenInfoDictionary"); -var ConnectionCosts = require("./ConnectionCosts"); -var UnknownDictionary = require("./UnknownDictionary"); +const doublearray = require("doublearray"); +const TokenInfoDictionary = require("./TokenInfoDictionary"); +const ConnectionCosts = require("./ConnectionCosts"); +const UnknownDictionary = require("./UnknownDictionary"); /** * Dictionaries container for Tokenizer diff --git a/src/dict/InvokeDefinitionMap.js b/src/dict/InvokeDefinitionMap.js index d97128b70274cb366f7027cbc34614726f6f595f..0425005b58fad7554a85f912e04dba745998f60f 100644 --- a/src/dict/InvokeDefinitionMap.js +++ b/src/dict/InvokeDefinitionMap.js @@ -17,8 +17,8 @@ "use strict"; -var ByteBuffer = require("../util/ByteBuffer"); -var CharacterClass = require("./CharacterClass"); +const ByteBuffer = require("../util/ByteBuffer"); +const CharacterClass = require("./CharacterClass"); /** * InvokeDefinitionMap represents invoke definition a part of char.def @@ -35,17 +35,17 @@ function InvokeDefinitionMap() { * @returns {InvokeDefinitionMap} */ InvokeDefinitionMap.load = function (invoke_def_buffer) { - var invoke_def = new InvokeDefinitionMap(); - var character_category_definition = []; + const invoke_def = new InvokeDefinitionMap(); + const character_category_definition = []; - var buffer = new ByteBuffer(invoke_def_buffer); - while (buffer.position + 1 < buffer.size()) { - var class_id = character_category_definition.length; - var is_always_invoke = buffer.get(); - var is_grouping = buffer.get(); - var max_length = buffer.getInt(); - var class_name = buffer.getString(); - character_category_definition.push(new CharacterClass(class_id, class_name, is_always_invoke, is_grouping, max_length)); + const buffer = new ByteBuffer(invoke_def_buffer); + while (buffer.position + 1 < buffer.size()) { + const class_id = character_category_definition.length; + const is_always_invoke = buffer.get(); + const is_grouping = buffer.get(); + const max_length = buffer.getInt(); + const class_name = buffer.getString(); + character_category_definition.push(new CharacterClass(class_id, class_name, is_always_invoke, is_grouping, max_length)); } invoke_def.init(character_category_definition); @@ -61,9 +61,9 @@ InvokeDefinitionMap.prototype.init = function (character_category_definition) { if (character_category_definition == null) { return; } - for (var i = 0; i < character_category_definition.length; i++) { - var character_class = character_category_definition[i]; - this.map[i] = character_class; + for (let i = 0; i < character_category_definition.length; i++) { + const character_class = character_category_definition[i]; + this.map[i] = character_class; this.lookup_table[character_class.class_name] = i; } }; @@ -83,8 +83,8 @@ InvokeDefinitionMap.prototype.getCharacterClass = function (class_id) { * @returns {number} class_id */ InvokeDefinitionMap.prototype.lookup = function (class_name) { - var class_id = this.lookup_table[class_name]; - if (class_id == null) { + const class_id = this.lookup_table[class_name]; + if (class_id == null) { return null; } return class_id; @@ -95,10 +95,10 @@ InvokeDefinitionMap.prototype.lookup = function (class_name) { * @returns {Uint8Array} */ InvokeDefinitionMap.prototype.toBuffer = function () { - var buffer = new ByteBuffer(); - for (var i = 0; i < this.map.length; i++) { - var char_class = this.map[i]; - buffer.put(char_class.is_always_invoke); + const buffer = new ByteBuffer(); + for (let i = 0; i < this.map.length; i++) { + const char_class = this.map[i]; + buffer.put(char_class.is_always_invoke); buffer.put(char_class.is_grouping); buffer.putInt(char_class.max_length); buffer.putString(char_class.class_name); diff --git a/src/dict/TokenInfoDictionary.js b/src/dict/TokenInfoDictionary.js index 2fccee7bf399abf070ca8adb1f634a141d3e351b..79d7a6cfc3b7a6be71bd786deabbe36c299ae376 100644 --- a/src/dict/TokenInfoDictionary.js +++ b/src/dict/TokenInfoDictionary.js @@ -17,7 +17,7 @@ "use strict"; -var ByteBuffer = require("../util/ByteBuffer"); +const ByteBuffer = require("../util/ByteBuffer"); /** * TokenInfoDictionary @@ -32,28 +32,28 @@ function TokenInfoDictionary() { // left_id right_id word_cost ... // ^ this position is token_info_id TokenInfoDictionary.prototype.buildDictionary = function (entries) { - var dictionary_entries = {}; // using as hashmap, string -> string (word_id -> surface_form) to build dictionary + const dictionary_entries = {}; // using as hashmap, string -> string (word_id -> surface_form) to build dictionary - for (var i = 0; i < entries.length; i++) { - var entry = entries[i]; + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; - if (entry.length < 4) { + if (entry.length < 4) { continue; } - var surface_form = entry[0]; - var left_id = entry[1]; - var right_id = entry[2]; - var word_cost = entry[3]; - var feature = entry.slice(4).join(","); // TODO Optimize + const surface_form = entry[0]; + const left_id = entry[1]; + const right_id = entry[2]; + const word_cost = entry[3]; + const feature = entry.slice(4).join(","); // TODO Optimize // Assertion if (!isFinite(left_id) || !isFinite(right_id) || !isFinite(word_cost)) { console.log(entry); } - var token_info_id = this.put(left_id, right_id, word_cost, surface_form, feature); - dictionary_entries[token_info_id] = surface_form; + const token_info_id = this.put(left_id, right_id, word_cost, surface_form, feature); + dictionary_entries[token_info_id] = surface_form; } // Remove last unused area @@ -64,10 +64,10 @@ TokenInfoDictionary.prototype.buildDictionary = function (entries) { }; TokenInfoDictionary.prototype.put = function (left_id, right_id, word_cost, surface_form, feature) { - var token_info_id = this.dictionary.position; - var pos_id = this.pos_buffer.position; + const token_info_id = this.dictionary.position; + const pos_id = this.pos_buffer.position; - this.dictionary.putShort(left_id); + this.dictionary.putShort(left_id); this.dictionary.putShort(right_id); this.dictionary.putShort(word_cost); this.dictionary.putInt(pos_id); @@ -77,8 +77,8 @@ TokenInfoDictionary.prototype.put = function (left_id, right_id, word_cost, surf }; TokenInfoDictionary.prototype.addMapping = function (source, target) { - var mapping = this.target_map[source]; - if (mapping == null) { + let mapping = this.target_map[source]; + if (mapping == null) { mapping = []; } mapping.push(target); @@ -87,15 +87,15 @@ TokenInfoDictionary.prototype.addMapping = function (source, target) { }; TokenInfoDictionary.prototype.targetMapToBuffer = function () { - var buffer = new ByteBuffer(); - var map_keys_size = Object.keys(this.target_map).length; - buffer.putInt(map_keys_size); - for (var key in this.target_map) { - var values = this.target_map[key]; // Array - var map_values_size = values.length; - buffer.putInt(parseInt(key)); + const buffer = new ByteBuffer(); + const map_keys_size = Object.keys(this.target_map).length; + buffer.putInt(map_keys_size); + for (let key in this.target_map) { + const values = this.target_map[key]; // Array + const map_values_size = values.length; + buffer.putInt(parseInt(key)); buffer.putInt(map_values_size); - for (var i = 0; i < values.length; i++) { + for (let i = 0; i < values.length; i++) { buffer.putInt(values[i]); } } @@ -116,19 +116,19 @@ TokenInfoDictionary.prototype.loadPosVector = function (array_buffer) { // from tid_map.dat TokenInfoDictionary.prototype.loadTargetMap = function (array_buffer) { - var buffer = new ByteBuffer(array_buffer); - buffer.position = 0; + const buffer = new ByteBuffer(array_buffer); + buffer.position = 0; this.target_map = {}; buffer.readInt(); // map_keys_size while (true) { if (buffer.buffer.length < buffer.position + 1) { break; } - var key = buffer.readInt(); - var map_values_size = buffer.readInt(); - for (var i = 0; i < map_values_size; i++) { - var value = buffer.readInt(); - this.addMapping(key, value); + const key = buffer.readInt(); + const map_values_size = buffer.readInt(); + for (let i = 0; i < map_values_size; i++) { + const value = buffer.readInt(); + this.addMapping(key, value); } } return this; @@ -140,13 +140,13 @@ TokenInfoDictionary.prototype.loadTargetMap = function (array_buffer) { * @returns {string} Features string concatenated by "," */ TokenInfoDictionary.prototype.getFeatures = function (token_info_id_str) { - var token_info_id = parseInt(token_info_id_str); - if (isNaN(token_info_id)) { + const token_info_id = parseInt(token_info_id_str); + if (isNaN(token_info_id)) { // TODO throw error return ""; } - var pos_id = this.dictionary.getInt(token_info_id + 6); - return this.pos_buffer.getString(pos_id); + const pos_id = this.dictionary.getInt(token_info_id + 6); + return this.pos_buffer.getString(pos_id); }; module.exports = TokenInfoDictionary; diff --git a/src/dict/UnknownDictionary.js b/src/dict/UnknownDictionary.js index 9814c77ca197812c97170970012e661b69c0db9a..f467dbf4df885cc145c344e818105b3ebb74b37b 100644 --- a/src/dict/UnknownDictionary.js +++ b/src/dict/UnknownDictionary.js @@ -17,9 +17,9 @@ "use strict"; -var TokenInfoDictionary = require("./TokenInfoDictionary"); -var CharacterDefinition = require("./CharacterDefinition"); -var ByteBuffer = require("../util/ByteBuffer"); +const TokenInfoDictionary = require("./TokenInfoDictionary"); +const CharacterDefinition = require("./CharacterDefinition"); +const ByteBuffer = require("../util/ByteBuffer"); /** * UnknownDictionary diff --git a/src/dict/builder/CharacterDefinitionBuilder.js b/src/dict/builder/CharacterDefinitionBuilder.js index 771e3c73c31c8f8f8da333bd40bf4c6f86727790..ec8446ef6cf38988a466016a8c2ebdec20b275a3 100644 --- a/src/dict/builder/CharacterDefinitionBuilder.js +++ b/src/dict/builder/CharacterDefinitionBuilder.js @@ -17,12 +17,12 @@ "use strict"; -var CharacterDefinition = require("../CharacterDefinition"); -var InvokeDefinitionMap = require("../InvokeDefinitionMap"); +const CharacterDefinition = require("../CharacterDefinition"); +const InvokeDefinitionMap = require("../InvokeDefinitionMap"); -var CATEGORY_DEF_PATTERN = /^(\w+)\s+(\d)\s+(\d)\s+(\d)/; -var CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; -var RANGE_CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})\.\.(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; +const CATEGORY_DEF_PATTERN = /^(\w+)\s+(\d)\s+(\d)\s+(\d)/; +const CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; +const RANGE_CATEGORY_MAPPING_PATTERN = /^(0x[0-9A-F]{4})\.\.(0x[0-9A-F]{4})(?:\s+([^#\s]+))(?:\s+([^#\s]+))*/; /** * CharacterDefinitionBuilder @@ -36,25 +36,25 @@ function CharacterDefinitionBuilder() { } CharacterDefinitionBuilder.prototype.putLine = function (line) { - var parsed_category_def = CATEGORY_DEF_PATTERN.exec(line); - if (parsed_category_def != null) { - var class_id = this.character_category_definition.length; - var char_class = CharacterDefinition.parseCharCategory(class_id, parsed_category_def); - if (char_class == null) { + const parsed_category_def = CATEGORY_DEF_PATTERN.exec(line); + if (parsed_category_def != null) { + const class_id = this.character_category_definition.length; + const char_class = CharacterDefinition.parseCharCategory(class_id, parsed_category_def); + if (char_class == null) { return; } this.character_category_definition.push(char_class); return; } - var parsed_category_mapping = CATEGORY_MAPPING_PATTERN.exec(line); - if (parsed_category_mapping != null) { - var mapping = CharacterDefinition.parseCategoryMapping(parsed_category_mapping); - this.category_mapping.push(mapping); + const parsed_category_mapping = CATEGORY_MAPPING_PATTERN.exec(line); + if (parsed_category_mapping != null) { + const mapping = CharacterDefinition.parseCategoryMapping(parsed_category_mapping); + this.category_mapping.push(mapping); } - var parsed_range_category_mapping = RANGE_CATEGORY_MAPPING_PATTERN.exec(line); - if (parsed_range_category_mapping != null) { - var range_mapping = CharacterDefinition.parseRangeCategoryMapping(parsed_range_category_mapping); - this.category_mapping.push(range_mapping); + const parsed_range_category_mapping = RANGE_CATEGORY_MAPPING_PATTERN.exec(line); + if (parsed_range_category_mapping != null) { + const range_mapping = CharacterDefinition.parseRangeCategoryMapping(parsed_range_category_mapping); + this.category_mapping.push(range_mapping); } }; diff --git a/src/dict/builder/ConnectionCostsBuilder.js b/src/dict/builder/ConnectionCostsBuilder.js index c09997a0c15b377c9d6e02e98afa9e7d04605edd..e499ac5a275c97e9cfdbd7badd8c05a83559a26f 100644 --- a/src/dict/builder/ConnectionCostsBuilder.js +++ b/src/dict/builder/ConnectionCostsBuilder.js @@ -17,7 +17,7 @@ "use strict"; -var ConnectionCosts = require("../ConnectionCosts"); +const ConnectionCosts = require("../ConnectionCosts"); /** * Builder class for constructing ConnectionCosts object @@ -30,11 +30,11 @@ function ConnectionCostsBuilder() { ConnectionCostsBuilder.prototype.putLine = function (line) { if (this.lines === 0) { - var dimensions = line.split(" "); - var forward_dimension = dimensions[0]; - var backward_dimension = dimensions[1]; + const dimensions = line.split(" "); + const forward_dimension = dimensions[0]; + const backward_dimension = dimensions[1]; - if (forward_dimension < 0 || backward_dimension < 0) { + if (forward_dimension < 0 || backward_dimension < 0) { throw "Parse error of matrix.def"; } @@ -43,17 +43,17 @@ ConnectionCostsBuilder.prototype.putLine = function (line) { return this; } - var costs = line.split(" "); + const costs = line.split(" "); - if (costs.length !== 3) { + if (costs.length !== 3) { return this; } - var forward_id = parseInt(costs[0]); - var backward_id = parseInt(costs[1]); - var cost = parseInt(costs[2]); + const forward_id = parseInt(costs[0]); + const backward_id = parseInt(costs[1]); + const cost = parseInt(costs[2]); - if (forward_id < 0 || backward_id < 0 || !isFinite(forward_id) || !isFinite(backward_id) || + if (forward_id < 0 || backward_id < 0 || !isFinite(forward_id) || !isFinite(backward_id) || this.connection_cost.forward_dimension <= forward_id || this.connection_cost.backward_dimension <= backward_id) { throw "Parse error of matrix.def"; } diff --git a/src/dict/builder/DictionaryBuilder.js b/src/dict/builder/DictionaryBuilder.js index 216c769f721c4d0513937f8b289ca4368ba6c526..b1f08243da88e90417db2b7f3263024488183b4a 100644 --- a/src/dict/builder/DictionaryBuilder.js +++ b/src/dict/builder/DictionaryBuilder.js @@ -17,12 +17,12 @@ "use strict"; -var doublearray = require("doublearray"); -var DynamicDictionaries = require("../DynamicDictionaries"); -var TokenInfoDictionary = require("../TokenInfoDictionary"); -var ConnectionCostsBuilder = require("./ConnectionCostsBuilder"); -var CharacterDefinitionBuilder = require("./CharacterDefinitionBuilder"); -var UnknownDictionary = require("../UnknownDictionary"); +const doublearray = require("doublearray"); +const DynamicDictionaries = require("../DynamicDictionaries"); +const TokenInfoDictionary = require("../TokenInfoDictionary"); +const ConnectionCostsBuilder = require("./ConnectionCostsBuilder"); +const CharacterDefinitionBuilder = require("./CharacterDefinitionBuilder"); +const UnknownDictionary = require("../UnknownDictionary"); /** * Build dictionaries (token info, connection costs) @@ -46,8 +46,8 @@ function DictionaryBuilder() { } DictionaryBuilder.prototype.addTokenInfoDictionary = function (line) { - var new_entry = line.split(","); - this.tid_entries.push(new_entry); + const new_entry = line.split(","); + this.tid_entries.push(new_entry); return this; }; @@ -75,10 +75,10 @@ DictionaryBuilder.prototype.putUnkDefLine = function (line) { }; DictionaryBuilder.prototype.build = function () { - var dictionaries = this.buildTokenInfoDictionary(); - var unknown_dictionary = this.buildUnknownDictionary(); + const dictionaries = this.buildTokenInfoDictionary(); + const unknown_dictionary = this.buildUnknownDictionary(); - return new DynamicDictionaries(dictionaries.trie, dictionaries.token_info_dictionary, this.cc_builder.build(), unknown_dictionary); + return new DynamicDictionaries(dictionaries.trie, dictionaries.token_info_dictionary, this.cc_builder.build(), unknown_dictionary); }; /** @@ -88,18 +88,18 @@ DictionaryBuilder.prototype.build = function () { */ DictionaryBuilder.prototype.buildTokenInfoDictionary = function () { - var token_info_dictionary = new TokenInfoDictionary(); + const token_info_dictionary = new TokenInfoDictionary(); - // using as hashmap, string -> string (word_id -> surface_form) to build dictionary - var dictionary_entries = token_info_dictionary.buildDictionary(this.tid_entries); + // using as hashmap, string -> string (word_id -> surface_form) to build dictionary + const dictionary_entries = token_info_dictionary.buildDictionary(this.tid_entries); - var trie = this.buildDoubleArray(); + const trie = this.buildDoubleArray(); - for (var token_info_id in dictionary_entries) { - var surface_form = dictionary_entries[token_info_id]; - var trie_id = trie.lookup(surface_form); + for (let token_info_id in dictionary_entries) { + const surface_form = dictionary_entries[token_info_id]; + const trie_id = trie.lookup(surface_form); - // Assertion + // Assertion // if (trie_id < 0) { // console.log("Not Found:" + surface_form); // } @@ -115,20 +115,20 @@ DictionaryBuilder.prototype.buildTokenInfoDictionary = function () { DictionaryBuilder.prototype.buildUnknownDictionary = function () { - var unk_dictionary = new UnknownDictionary(); + const unk_dictionary = new UnknownDictionary(); - // using as hashmap, string -> string (word_id -> surface_form) to build dictionary - var dictionary_entries = unk_dictionary.buildDictionary(this.unk_entries); + // using as hashmap, string -> string (word_id -> surface_form) to build dictionary + const dictionary_entries = unk_dictionary.buildDictionary(this.unk_entries); - var char_def = this.cd_builder.build(); // Create CharacterDefinition + const char_def = this.cd_builder.build(); // Create CharacterDefinition unk_dictionary.characterDefinition(char_def); - for (var token_info_id in dictionary_entries) { - var class_name = dictionary_entries[token_info_id]; - var class_id = char_def.invoke_definition_map.lookup(class_name); + for (let token_info_id in dictionary_entries) { + const class_name = dictionary_entries[token_info_id]; + const class_id = char_def.invoke_definition_map.lookup(class_name); - // Assertion + // Assertion // if (trie_id < 0) { // console.log("Not Found:" + surface_form); // } @@ -145,14 +145,14 @@ DictionaryBuilder.prototype.buildUnknownDictionary = function () { * @returns {DoubleArray} Double-Array trie */ DictionaryBuilder.prototype.buildDoubleArray = function () { - var trie_id = 0; - var words = this.tid_entries.map(function (entry) { - var surface_form = entry[0]; - return { k: surface_form, v: trie_id++ }; - }); - - var builder = doublearray.builder(1024 * 1024); - return builder.build(words); + let trie_id = 0; + const words = this.tid_entries.map(function (entry) { + const surface_form = entry[0]; + return {k: surface_form, v: trie_id++}; + }); + + const builder = doublearray.builder(1024 * 1024); + return builder.build(words); }; module.exports = DictionaryBuilder; diff --git a/src/kuromoji.js b/src/kuromoji.js index 4820d23e6a257e736cfa958b345cab5c03e3b4e5..c2d548dae6e45a21973d12eea893469d6aff742e 100644 --- a/src/kuromoji.js +++ b/src/kuromoji.js @@ -17,17 +17,17 @@ "use strict"; -var TokenizerBuilder = require("./TokenizerBuilder"); -var DictionaryBuilder = require("./dict/builder/DictionaryBuilder"); +const TokenizerBuilder = require("./TokenizerBuilder"); +const DictionaryBuilder = require("./dict/builder/DictionaryBuilder"); // Public methods -var kuromoji = { - builder: function (option) { - return new TokenizerBuilder(option); - }, - dictionaryBuilder: function () { - return new DictionaryBuilder(); - } +const kuromoji = { + builder: function (option) { + return new TokenizerBuilder(option); + }, + dictionaryBuilder: function () { + return new DictionaryBuilder(); + } }; module.exports = kuromoji; diff --git a/src/loader/BrowserDictionaryLoader.js b/src/loader/BrowserDictionaryLoader.js index 04bfdcd1c16b66960b3377152afb357e268ea872..d0473f53250179e9dabfc81e8fb2b206a7823327 100644 --- a/src/loader/BrowserDictionaryLoader.js +++ b/src/loader/BrowserDictionaryLoader.js @@ -17,8 +17,8 @@ "use strict"; -var zlib = require("zlibjs/bin/gunzip.min.js"); -var DictionaryLoader = require("./DictionaryLoader"); +const fflate = require("fflate"); +const DictionaryLoader = require("./DictionaryLoader"); /** * BrowserDictionaryLoader inherits DictionaryLoader, using jQuery XHR for download @@ -37,24 +37,17 @@ BrowserDictionaryLoader.prototype = Object.create(DictionaryLoader.prototype); * @param {BrowserDictionaryLoader~onLoad} callback Callback function */ BrowserDictionaryLoader.prototype.loadArrayBuffer = function (url, callback) { - var xhr = new XMLHttpRequest(); - xhr.open("GET", url, true); - xhr.responseType = "arraybuffer"; - xhr.onload = function () { - if (this.status > 0 && this.status !== 200) { - callback(xhr.statusText, null); - return; - } - var arraybuffer = this.response; - - var gz = new zlib.Zlib.Gunzip(new Uint8Array(arraybuffer)); - var typed_array = gz.decompress(); - callback(null, typed_array.buffer); - }; - xhr.onerror = function (err) { - callback(err, null); - }; - xhr.send(); + fetch(url).then(function (response) { + if (!response.ok) { + callback(response.statusText, null); + } + response.arrayBuffer().then(function (arraybuffer) { + const gz = fflate.gunzipSync(new Uint8Array(arraybuffer)); + callback(null, gz.buffer); + }); + }).catch(function (exception) { + callback(exception, null); + }); }; /** diff --git a/src/loader/DictionaryLoader.js b/src/loader/DictionaryLoader.js index 5f88c0b7f9a786dd8c072a7b84ae86a6f31412cb..402e9271c677f549e9521b959f34e1043faaf7a6 100644 --- a/src/loader/DictionaryLoader.js +++ b/src/loader/DictionaryLoader.js @@ -17,9 +17,8 @@ "use strict"; -var path = require("path"); -var async = require("async"); -var DynamicDictionaries = require("../dict/DynamicDictionaries"); +const async = require("async"); +const DynamicDictionaries = require("../dict/DynamicDictionaries"); /** * DictionaryLoader base constructor @@ -40,15 +39,15 @@ DictionaryLoader.prototype.loadArrayBuffer = function (file, callback) { * @param {DictionaryLoader~onLoad} load_callback Callback function called after loaded */ DictionaryLoader.prototype.load = function (load_callback) { - var dic = this.dic; - var dic_path = this.dic_path; - var loadArrayBuffer = this.loadArrayBuffer; + const dic = this.dic; + const dic_path = this.dic_path; + const loadArrayBuffer = this.loadArrayBuffer; - async.parallel([ + async.parallel([ // Trie function (callback) { async.map([ "base.dat.gz", "check.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -58,17 +57,17 @@ DictionaryLoader.prototype.load = function (load_callback) { if(err) { return callback(err); } - var base_buffer = new Int32Array(buffers[0]); - var check_buffer = new Int32Array(buffers[1]); + const base_buffer = new Int32Array(buffers[0]); + const check_buffer = new Int32Array(buffers[1]); - dic.loadTrie(base_buffer, check_buffer); + dic.loadTrie(base_buffer, check_buffer); callback(null); }); }, // Token info dictionaries function (callback) { async.map([ "tid.dat.gz", "tid_pos.dat.gz", "tid_map.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -78,29 +77,29 @@ DictionaryLoader.prototype.load = function (load_callback) { if(err) { return callback(err); } - var token_info_buffer = new Uint8Array(buffers[0]); - var pos_buffer = new Uint8Array(buffers[1]); - var target_map_buffer = new Uint8Array(buffers[2]); + const token_info_buffer = new Uint8Array(buffers[0]); + const pos_buffer = new Uint8Array(buffers[1]); + const target_map_buffer = new Uint8Array(buffers[2]); - dic.loadTokenInfoDictionaries(token_info_buffer, pos_buffer, target_map_buffer); + dic.loadTokenInfoDictionaries(token_info_buffer, pos_buffer, target_map_buffer); callback(null); }); }, // Connection cost matrix function (callback) { - loadArrayBuffer(path.join(dic_path, "cc.dat.gz"), function (err, buffer) { + loadArrayBuffer(dic_path + "cc.dat.gz", function (err, buffer) { if(err) { return callback(err); } - var cc_buffer = new Int16Array(buffer); - dic.loadConnectionCosts(cc_buffer); + const cc_buffer = new Int16Array(buffer); + dic.loadConnectionCosts(cc_buffer); callback(null); }); }, // Unknown dictionaries function (callback) { async.map([ "unk.dat.gz", "unk_pos.dat.gz", "unk_map.dat.gz", "unk_char.dat.gz", "unk_compat.dat.gz", "unk_invoke.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -110,14 +109,14 @@ DictionaryLoader.prototype.load = function (load_callback) { if(err) { return callback(err); } - var unk_buffer = new Uint8Array(buffers[0]); - var unk_pos_buffer = new Uint8Array(buffers[1]); - var unk_map_buffer = new Uint8Array(buffers[2]); - var cat_map_buffer = new Uint8Array(buffers[3]); - var compat_cat_map_buffer = new Uint32Array(buffers[4]); - var invoke_def_buffer = new Uint8Array(buffers[5]); + const unk_buffer = new Uint8Array(buffers[0]); + const unk_pos_buffer = new Uint8Array(buffers[1]); + const unk_map_buffer = new Uint8Array(buffers[2]); + const cat_map_buffer = new Uint8Array(buffers[3]); + const compat_cat_map_buffer = new Uint32Array(buffers[4]); + const invoke_def_buffer = new Uint8Array(buffers[5]); - dic.loadUnknownDictionaries(unk_buffer, unk_pos_buffer, unk_map_buffer, cat_map_buffer, compat_cat_map_buffer, invoke_def_buffer); + dic.loadUnknownDictionaries(unk_buffer, unk_pos_buffer, unk_map_buffer, cat_map_buffer, compat_cat_map_buffer, invoke_def_buffer); // dic.loadUnknownDictionaries(char_buffer, unk_buffer); callback(null); }); diff --git a/src/loader/NodeDictionaryLoader.js b/src/loader/NodeDictionaryLoader.js deleted file mode 100644 index 26eb79249121efe39bd5ae77c17e1caa197fb4ce..0000000000000000000000000000000000000000 diff --git a/src/util/ByteBuffer.js b/src/util/ByteBuffer.js index 1efa453a8cce6fe697653be424782405468dbbf2..8148d612cfec33aa1ef02eb0963582805d69ac6c 100644 --- a/src/util/ByteBuffer.js +++ b/src/util/ByteBuffer.js @@ -23,63 +23,64 @@ * @param {String} str UTF-16 string to convert * @return {Uint8Array} Byte sequence encoded by UTF-8 */ -var stringToUtf8Bytes = function (str) { - - // Max size of 1 character is 4 bytes - var bytes = new Uint8Array(str.length * 4); - - var i = 0, j = 0; - - while (i < str.length) { - var unicode_code; - - var utf16_code = str.charCodeAt(i++); - if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { - // surrogate pair - var upper = utf16_code; // high surrogate - var lower = str.charCodeAt(i++); // low surrogate - - if (lower >= 0xDC00 && lower <= 0xDFFF) { - unicode_code = - (upper - 0xD800) * (1 << 10) + (1 << 16) + - (lower - 0xDC00); - } else { - // malformed surrogate pair - return null; - } - } else { - // not surrogate code - unicode_code = utf16_code; - } +const stringToUtf8Bytes = function (str) { + + // Max size of 1 character is 4 bytes + const bytes = new Uint8Array(str.length * 4); + + let i = 0; + let j = 0; + + while (i < str.length) { + let unicode_code; + + const utf16_code = str.charCodeAt(i++); + if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { + // surrogate pair + const upper = utf16_code; // high surrogate + const lower = str.charCodeAt(i++); // low surrogate + + if (lower >= 0xDC00 && lower <= 0xDFFF) { + unicode_code = + (upper - 0xD800) * (1 << 10) + (1 << 16) + + (lower - 0xDC00); + } else { + // malformed surrogate pair + return null; + } + } else { + // not surrogate code + unicode_code = utf16_code; + } - if (unicode_code < 0x80) { - // 1-byte - bytes[j++] = unicode_code; + if (unicode_code < 0x80) { + // 1-byte + bytes[j++] = unicode_code; - } else if (unicode_code < (1 << 11)) { - // 2-byte - bytes[j++] = (unicode_code >>> 6) | 0xC0; - bytes[j++] = (unicode_code & 0x3F) | 0x80; + } else if (unicode_code < (1 << 11)) { + // 2-byte + bytes[j++] = (unicode_code >>> 6) | 0xC0; + bytes[j++] = (unicode_code & 0x3F) | 0x80; - } else if (unicode_code < (1 << 16)) { - // 3-byte - bytes[j++] = (unicode_code >>> 12) | 0xE0; - bytes[j++] = ((unicode_code >> 6) & 0x3f) | 0x80; - bytes[j++] = (unicode_code & 0x3F) | 0x80; + } else if (unicode_code < (1 << 16)) { + // 3-byte + bytes[j++] = (unicode_code >>> 12) | 0xE0; + bytes[j++] = ((unicode_code >> 6) & 0x3f) | 0x80; + bytes[j++] = (unicode_code & 0x3F) | 0x80; - } else if (unicode_code < (1 << 21)) { - // 4-byte - bytes[j++] = (unicode_code >>> 18) | 0xF0; - bytes[j++] = ((unicode_code >> 12) & 0x3F) | 0x80; - bytes[j++] = ((unicode_code >> 6) & 0x3F) | 0x80; - bytes[j++] = (unicode_code & 0x3F) | 0x80; + } else if (unicode_code < (1 << 21)) { + // 4-byte + bytes[j++] = (unicode_code >>> 18) | 0xF0; + bytes[j++] = ((unicode_code >> 12) & 0x3F) | 0x80; + bytes[j++] = ((unicode_code >> 6) & 0x3F) | 0x80; + bytes[j++] = (unicode_code & 0x3F) | 0x80; - } else { - // malformed UCS4 code - } + } else { + // malformed UCS4 code } + } - return bytes.subarray(0, j); + return bytes.subarray(0, j); }; /** @@ -88,48 +89,48 @@ var stringToUtf8Bytes = function (str) { * @param {Array} bytes UTF-8 byte sequence to convert * @return {String} String encoded by UTF-16 */ -var utf8BytesToString = function (bytes) { - - var str = ""; - var code, b1, b2, b3, b4, upper, lower; - var i = 0; - - while (i < bytes.length) { - - b1 = bytes[i++]; - - if (b1 < 0x80) { - // 1 byte - code = b1; - } else if ((b1 >> 5) === 0x06) { - // 2 bytes - b2 = bytes[i++]; - code = ((b1 & 0x1f) << 6) | (b2 & 0x3f); - } else if ((b1 >> 4) === 0x0e) { - // 3 bytes - b2 = bytes[i++]; - b3 = bytes[i++]; - code = ((b1 & 0x0f) << 12) | ((b2 & 0x3f) << 6) | (b3 & 0x3f); - } else { - // 4 bytes - b2 = bytes[i++]; - b3 = bytes[i++]; - b4 = bytes[i++]; - code = ((b1 & 0x07) << 18) | ((b2 & 0x3f) << 12) | ((b3 & 0x3f) << 6) | (b4 & 0x3f); - } +const utf8BytesToString = function (bytes) { + + let str = ""; + let code, b1, b2, b3, b4, upper, lower; + let i = 0; + + while (i < bytes.length) { + + b1 = bytes[i++]; + + if (b1 < 0x80) { + // 1 byte + code = b1; + } else if ((b1 >> 5) === 0x06) { + // 2 bytes + b2 = bytes[i++]; + code = ((b1 & 0x1f) << 6) | (b2 & 0x3f); + } else if ((b1 >> 4) === 0x0e) { + // 3 bytes + b2 = bytes[i++]; + b3 = bytes[i++]; + code = ((b1 & 0x0f) << 12) | ((b2 & 0x3f) << 6) | (b3 & 0x3f); + } else { + // 4 bytes + b2 = bytes[i++]; + b3 = bytes[i++]; + b4 = bytes[i++]; + code = ((b1 & 0x07) << 18) | ((b2 & 0x3f) << 12) | ((b3 & 0x3f) << 6) | (b4 & 0x3f); + } - if (code < 0x10000) { - str += String.fromCharCode(code); - } else { - // surrogate pair - code -= 0x10000; - upper = (0xD800 | (code >> 10)); - lower = (0xDC00 | (code & 0x3FF)); - str += String.fromCharCode(upper, lower); - } + if (code < 0x10000) { + str += String.fromCharCode(code); + } else { + // surrogate pair + code -= 0x10000; + upper = (0xD800 | (code >> 10)); + lower = (0xDC00 | (code & 0x3FF)); + str += String.fromCharCode(upper, lower); } + } - return str; + return str; }; /** @@ -138,8 +139,8 @@ var utf8BytesToString = function (bytes) { * @constructor */ function ByteBuffer(arg) { - var initial_size; - if (arg == null) { + let initial_size; + if (arg == null) { initial_size = 1024 * 1024; } else if (typeof arg === "number") { initial_size = arg; @@ -161,8 +162,8 @@ ByteBuffer.prototype.size = function () { }; ByteBuffer.prototype.reallocate = function () { - var new_array = new Uint8Array(this.buffer.length * 2); - new_array.set(this.buffer); + const new_array = new Uint8Array(this.buffer.length * 2); + new_array.set(this.buffer); this.buffer = new_array; }; @@ -194,9 +195,9 @@ ByteBuffer.prototype.putShort = function (num) { if (0xFFFF < num) { throw num + " is over short value"; } - var lower = (0x00FF & num); - var upper = (0xFF00 & num) >> 8; - this.put(lower); + const lower = (0x00FF & num); + const upper = (0xFF00 & num) >> 8; + this.put(lower); this.put(upper); }; @@ -209,10 +210,10 @@ ByteBuffer.prototype.getShort = function (index) { if (this.buffer.length < index + 2) { return 0; } - var lower = this.buffer[index]; - var upper = this.buffer[index + 1]; - var value = (upper << 8) + lower; - if (value & 0x8000) { + const lower = this.buffer[index]; + const upper = this.buffer[index + 1]; + let value = (upper << 8) + lower; + if (value & 0x8000) { value = -((value - 1) ^ 0xFFFF); } return value; @@ -223,11 +224,11 @@ ByteBuffer.prototype.putInt = function (num) { if (0xFFFFFFFF < num) { throw num + " is over integer value"; } - var b0 = (0x000000FF & num); - var b1 = (0x0000FF00 & num) >> 8; - var b2 = (0x00FF0000 & num) >> 16; - var b3 = (0xFF000000 & num) >> 24; - this.put(b0); + const b0 = (0x000000FF & num); + const b1 = (0x0000FF00 & num) >> 8; + const b2 = (0x00FF0000 & num) >> 16; + const b3 = (0xFF000000 & num) >> 24; + this.put(b0); this.put(b1); this.put(b2); this.put(b3); @@ -242,23 +243,23 @@ ByteBuffer.prototype.getInt = function (index) { if (this.buffer.length < index + 4) { return 0; } - var b0 = this.buffer[index]; - var b1 = this.buffer[index + 1]; - var b2 = this.buffer[index + 2]; - var b3 = this.buffer[index + 3]; + const b0 = this.buffer[index]; + const b1 = this.buffer[index + 1]; + const b2 = this.buffer[index + 2]; + const b3 = this.buffer[index + 3]; - return (b3 << 24) + (b2 << 16) + (b1 << 8) + b0; + return (b3 << 24) + (b2 << 16) + (b1 << 8) + b0; }; ByteBuffer.prototype.readInt = function () { - var pos = this.position; - this.position += 4; + const pos = this.position; + this.position += 4; return this.getInt(pos); }; ByteBuffer.prototype.putString = function (str) { - var bytes = stringToUtf8Bytes(str); - for (var i = 0; i < bytes.length; i++) { + const bytes = stringToUtf8Bytes(str); + for (let i = 0; i < bytes.length; i++) { this.put(bytes[i]); } // put null character as terminal character @@ -266,9 +267,9 @@ ByteBuffer.prototype.putString = function (str) { }; ByteBuffer.prototype.getString = function (index) { - var buf = [], - ch; - if (index == null) { + const buf = []; + let ch; + if (index == null) { index = this.position; } while (true) { diff --git a/src/util/IpadicFormatter.js b/src/util/IpadicFormatter.js index 285e05c13b259dc75a473bf3190df90d3762fb56..5578e4a6a5a9d0a3272cc39a384e58be908feff3 100644 --- a/src/util/IpadicFormatter.js +++ b/src/util/IpadicFormatter.js @@ -25,8 +25,8 @@ function IpadicFormatter() { } IpadicFormatter.prototype.formatEntry = function (word_id, position, type, features) { - var token = {}; - token.word_id = word_id; + const token = {}; + token.word_id = word_id; token.word_type = type; token.word_position = position; @@ -45,8 +45,8 @@ IpadicFormatter.prototype.formatEntry = function (word_id, position, type, featu }; IpadicFormatter.prototype.formatUnknownEntry = function (word_id, position, type, features, surface_form) { - var token = {}; - token.word_id = word_id; + const token = {}; + token.word_id = word_id; token.word_type = type; token.word_position = position; diff --git a/src/util/SurrogateAwareString.js b/src/util/SurrogateAwareString.js index bfd88ceaa360567e582534805675ef816d8d69a3..0ef2ee2e1b8f8125e9f3c8858c722339b288e114 100644 --- a/src/util/SurrogateAwareString.js +++ b/src/util/SurrogateAwareString.js @@ -26,9 +26,9 @@ function SurrogateAwareString(str) { this.str = str; this.index_mapping = []; - for (var pos = 0; pos < str.length; pos++) { - var ch = str.charAt(pos); - this.index_mapping.push(pos); + for (let pos = 0; pos < str.length; pos++) { + const ch = str.charAt(pos); + this.index_mapping.push(pos); if (SurrogateAwareString.isSurrogatePair(ch)) { pos++; } @@ -41,18 +41,18 @@ SurrogateAwareString.prototype.slice = function (index) { if (this.index_mapping.length <= index) { return ""; } - var surrogate_aware_index = this.index_mapping[index]; - return this.str.slice(surrogate_aware_index); + const surrogate_aware_index = this.index_mapping[index]; + return this.str.slice(surrogate_aware_index); }; SurrogateAwareString.prototype.charAt = function (index) { if (this.str.length <= index) { return ""; } - var surrogate_aware_start_index = this.index_mapping[index]; - var surrogate_aware_end_index = this.index_mapping[index + 1]; + const surrogate_aware_start_index = this.index_mapping[index]; + const surrogate_aware_end_index = this.index_mapping[index + 1]; - if (surrogate_aware_end_index == null) { + if (surrogate_aware_end_index == null) { return this.str.slice(surrogate_aware_start_index); } return this.str.slice(surrogate_aware_start_index, surrogate_aware_end_index); @@ -62,10 +62,10 @@ SurrogateAwareString.prototype.charCodeAt = function (index) { if (this.index_mapping.length <= index) { return NaN; } - var surrogate_aware_index = this.index_mapping[index]; - var upper = this.str.charCodeAt(surrogate_aware_index); - var lower; - if (upper >= 0xD800 && upper <= 0xDBFF && surrogate_aware_index < this.str.length) { + const surrogate_aware_index = this.index_mapping[index]; + const upper = this.str.charCodeAt(surrogate_aware_index); + let lower; + if (upper >= 0xD800 && upper <= 0xDBFF && surrogate_aware_index < this.str.length) { lower = this.str.charCodeAt(surrogate_aware_index + 1); if (lower >= 0xDC00 && lower <= 0xDFFF) { return (upper - 0xD800) * 0x400 + lower - 0xDC00 + 0x10000; @@ -79,8 +79,8 @@ SurrogateAwareString.prototype.toString = function () { }; SurrogateAwareString.isSurrogatePair = function (ch) { - var utf16_code = ch.charCodeAt(0); - if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { + const utf16_code = ch.charCodeAt(0); + if (utf16_code >= 0xD800 && utf16_code <= 0xDBFF) { // surrogate pair return true; } else { diff --git a/src/viterbi/ViterbiBuilder.js b/src/viterbi/ViterbiBuilder.js index b5b8a5f4f37beb40e4395af401dcd87b8c853123..bcb401cdeb2ceba7d952202736455d2ada876d67 100644 --- a/src/viterbi/ViterbiBuilder.js +++ b/src/viterbi/ViterbiBuilder.js @@ -17,9 +17,9 @@ "use strict"; -var ViterbiNode = require("./ViterbiNode"); -var ViterbiLattice = require("./ViterbiLattice"); -var SurrogateAwareString = require("../util/SurrogateAwareString"); +const ViterbiNode = require("./ViterbiNode"); +const ViterbiLattice = require("./ViterbiLattice"); +const SurrogateAwareString = require("../util/SurrogateAwareString"); /** * ViterbiBuilder builds word lattice (ViterbiLattice) @@ -38,22 +38,22 @@ function ViterbiBuilder(dic) { * @returns {ViterbiLattice} Word lattice */ ViterbiBuilder.prototype.build = function (sentence_str) { - var lattice = new ViterbiLattice(); - var sentence = new SurrogateAwareString(sentence_str); + const lattice = new ViterbiLattice(); + const sentence = new SurrogateAwareString(sentence_str); - var key, trie_id, left_id, right_id, word_cost; - for (var pos = 0; pos < sentence.length; pos++) { - var tail = sentence.slice(pos); - var vocabulary = this.trie.commonPrefixSearch(tail); - for (var n = 0; n < vocabulary.length; n++) { // Words in dictionary do not have surrogate pair (only UCS2 set) + let key, trie_id, left_id, right_id, word_cost; + for (let pos = 0; pos < sentence.length; pos++) { + const tail = sentence.slice(pos); + const vocabulary = this.trie.commonPrefixSearch(tail); + for (let n = 0; n < vocabulary.length; n++) { // Words in dictionary do not have surrogate pair (only UCS2 set) trie_id = vocabulary[n].v; key = vocabulary[n].k; - var token_info_ids = this.token_info_dictionary.target_map[trie_id]; - for (var i = 0; i < token_info_ids.length; i++) { - var token_info_id = parseInt(token_info_ids[i]); + const token_info_ids = this.token_info_dictionary.target_map[trie_id]; + for (let i = 0; i < token_info_ids.length; i++) { + const token_info_id = parseInt(token_info_ids[i]); - left_id = this.token_info_dictionary.dictionary.getShort(token_info_id); + left_id = this.token_info_dictionary.dictionary.getShort(token_info_id); right_id = this.token_info_dictionary.dictionary.getShort(token_info_id + 2); word_cost = this.token_info_dictionary.dictionary.getShort(token_info_id + 4); @@ -63,28 +63,28 @@ ViterbiBuilder.prototype.build = function (sentence_str) { } // Unknown word processing - var surrogate_aware_tail = new SurrogateAwareString(tail); - var head_char = new SurrogateAwareString(surrogate_aware_tail.charAt(0)); - var head_char_class = this.unknown_dictionary.lookup(head_char.toString()); - if (vocabulary == null || vocabulary.length === 0 || head_char_class.is_always_invoke === 1) { + const surrogate_aware_tail = new SurrogateAwareString(tail); + const head_char = new SurrogateAwareString(surrogate_aware_tail.charAt(0)); + const head_char_class = this.unknown_dictionary.lookup(head_char.toString()); + if (vocabulary == null || vocabulary.length === 0 || head_char_class.is_always_invoke === 1) { // Process unknown word key = head_char; if (head_char_class.is_grouping === 1 && 1 < surrogate_aware_tail.length) { - for (var k = 1; k < surrogate_aware_tail.length; k++) { - var next_char = surrogate_aware_tail.charAt(k); - var next_char_class = this.unknown_dictionary.lookup(next_char); - if (head_char_class.class_name !== next_char_class.class_name) { + for (let k = 1; k < surrogate_aware_tail.length; k++) { + const next_char = surrogate_aware_tail.charAt(k); + const next_char_class = this.unknown_dictionary.lookup(next_char); + if (head_char_class.class_name !== next_char_class.class_name) { break; } key += next_char; } } - var unk_ids = this.unknown_dictionary.target_map[head_char_class.class_id]; - for (var j = 0; j < unk_ids.length; j++) { - var unk_id = parseInt(unk_ids[j]); + const unk_ids = this.unknown_dictionary.target_map[head_char_class.class_id]; + for (let j = 0; j < unk_ids.length; j++) { + const unk_id = parseInt(unk_ids[j]); - left_id = this.unknown_dictionary.dictionary.getShort(unk_id); + left_id = this.unknown_dictionary.dictionary.getShort(unk_id); right_id = this.unknown_dictionary.dictionary.getShort(unk_id + 2); word_cost = this.unknown_dictionary.dictionary.getShort(unk_id + 4); diff --git a/src/viterbi/ViterbiLattice.js b/src/viterbi/ViterbiLattice.js index 0ab121d22db35150ef92208fd205049702f7df51..4b5c314fae31704128cfc919983243aeda06ec12 100644 --- a/src/viterbi/ViterbiLattice.js +++ b/src/viterbi/ViterbiLattice.js @@ -17,7 +17,7 @@ "use strict"; -var ViterbiNode = require("./ViterbiNode"); +const ViterbiNode = require("./ViterbiNode"); /** * ViterbiLattice is a lattice in Viterbi algorithm @@ -34,13 +34,13 @@ function ViterbiLattice() { * @param {ViterbiNode} node */ ViterbiLattice.prototype.append = function (node) { - var last_pos = node.start_pos + node.length - 1; - if (this.eos_pos < last_pos) { + const last_pos = node.start_pos + node.length - 1; + if (this.eos_pos < last_pos) { this.eos_pos = last_pos; } - var prev_nodes = this.nodes_end_at[last_pos]; - if (prev_nodes == null) { + let prev_nodes = this.nodes_end_at[last_pos]; + if (prev_nodes == null) { prev_nodes = []; } prev_nodes.push(node); @@ -52,8 +52,8 @@ ViterbiLattice.prototype.append = function (node) { * Set ends with EOS (End of Statement) */ ViterbiLattice.prototype.appendEos = function () { - var last_index = this.nodes_end_at.length; - this.eos_pos++; + const last_index = this.nodes_end_at.length; + this.eos_pos++; this.nodes_end_at[last_index] = [ new ViterbiNode(-1, 0, this.eos_pos, 0, "EOS", 0, 0, "") ]; }; diff --git a/src/viterbi/ViterbiSearcher.js b/src/viterbi/ViterbiSearcher.js index 832e55432b1b461afa61fe4ecf29d9f5d15419e9..454eb544215581c9b293bd11dd3a3791510ef1ed 100644 --- a/src/viterbi/ViterbiSearcher.js +++ b/src/viterbi/ViterbiSearcher.js @@ -37,27 +37,27 @@ ViterbiSearcher.prototype.search = function (lattice) { }; ViterbiSearcher.prototype.forward = function (lattice) { - var i, j, k; - for (i = 1; i <= lattice.eos_pos; i++) { - var nodes = lattice.nodes_end_at[i]; - if (nodes == null) { + let i, j, k; + for (i = 1; i <= lattice.eos_pos; i++) { + const nodes = lattice.nodes_end_at[i]; + if (nodes == null) { continue; } for (j = 0; j < nodes.length; j++) { - var node = nodes[j]; - var cost = Number.MAX_VALUE; - var shortest_prev_node; + const node = nodes[j]; + let cost = Number.MAX_VALUE; + let shortest_prev_node; - var prev_nodes = lattice.nodes_end_at[node.start_pos - 1]; - if (prev_nodes == null) { + const prev_nodes = lattice.nodes_end_at[node.start_pos - 1]; + if (prev_nodes == null) { // TODO process unknown words (repair word lattice) continue; } for (k = 0; k < prev_nodes.length; k++) { - var prev_node = prev_nodes[k]; + const prev_node = prev_nodes[k]; - var edge_cost; - if (node.left_id == null || prev_node.right_id == null) { + let edge_cost; + if (node.left_id == null || prev_node.right_id == null) { // TODO assert console.log("Left or right is null"); edge_cost = 0; @@ -65,8 +65,8 @@ ViterbiSearcher.prototype.forward = function (lattice) { edge_cost = this.connection_costs.get(prev_node.right_id, node.left_id); } - var _cost = prev_node.shortest_cost + edge_cost + node.cost; - if (_cost < cost) { + const _cost = prev_node.shortest_cost + edge_cost + node.cost; + if (_cost < cost) { shortest_prev_node = prev_node; cost = _cost; } @@ -80,11 +80,11 @@ ViterbiSearcher.prototype.forward = function (lattice) { }; ViterbiSearcher.prototype.backward = function (lattice) { - var shortest_path = []; - var eos = lattice.nodes_end_at[lattice.nodes_end_at.length - 1][0]; + const shortest_path = []; + const eos = lattice.nodes_end_at[lattice.nodes_end_at.length - 1][0]; - var node_back = eos.prev; - if (node_back == null) { + let node_back = eos.prev; + if (node_back == null) { return []; } while (node_back.type !== "BOS") {