diff --git a/build/kuromoji.js b/build/kuromoji.js index f0f4ae9183ff8965fda64a2042f29936f76506d1..8912a754d184742d2768854c7bba83d66f9ff95f 100644 --- a/build/kuromoji.js +++ b/build/kuromoji.js @@ -1,5 +1,5 @@ -(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.kuromoji = f()}})(function(){var define,module,exports;return (function(){function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o -1 && value % 1 == 0 && value < length); + (type == 'number' || + (type != 'symbol' && reIsUint.test(value))) && + (value > -1 && value % 1 == 0 && value < length); } /** `Object#toString` result references. */ @@ -755,6 +758,14 @@ var freeProcess = moduleExports$1 && freeGlobal.process; /** Used to access faster Node.js helpers. */ var nodeUtil = (function() { try { + // Use `util.types` for Node.js 10+. + var types = freeModule$1 && freeModule$1.require && freeModule$1.require('util').types; + + if (types) { + return types; + } + + // Legacy `process.binding('util')` for Node.js < 10. return freeProcess && freeProcess.binding && freeProcess.binding('util'); } catch (e) {} }()); @@ -939,6 +950,9 @@ function createObjectIterator(obj) { var len = okeys.length; return function next() { var key = okeys[++i]; + if (key === '__proto__') { + return next(); + } return i < len ? {value: obj[key], key: key} : null; }; } @@ -970,6 +984,7 @@ function _eachOfLimit(limit) { var nextElem = iterator(obj); var done = false; var running = 0; + var looping = false; function iterateeCallback(err, value) { running -= 1; @@ -981,12 +996,13 @@ function _eachOfLimit(limit) { done = true; return callback(null); } - else { + else if (!looping) { replenish(); } } function replenish () { + looping = true; while (running < limit && !done) { var elem = nextElem(); if (elem === null) { @@ -999,6 +1015,7 @@ function _eachOfLimit(limit) { running += 1; iteratee(elem.value, elem.key, onlyOnce(iterateeCallback)); } + looping = false; } replenish(); @@ -3819,7 +3836,7 @@ function memoize(fn, hasher) { /** * Calls `callback` on a later loop around the event loop. In Node.js this just - * calls `process.nextTicl`. In the browser it will use `setImmediate` if + * calls `process.nextTick`. In the browser it will use `setImmediate` if * available, otherwise `setTimeout(callback, 0)`, which means other higher * priority events may precede the execution of `callback`. * @@ -5596,8 +5613,8 @@ Object.defineProperty(exports, '__esModule', { value: true }); }))); -}).call(this,require('_process'),typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) -},{"_process":4}],2:[function(require,module,exports){ +}).call(this)}).call(this,require('_process'),typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {},require("timers").setImmediate) +},{"_process":3,"timers":4}],2:[function(require,module,exports){ // Copyright (c) 2014 Takuya Asano All Rights Reserved. (function () { @@ -6391,234 +6408,6 @@ Object.defineProperty(exports, '__esModule', { value: true }); })(); },{}],3:[function(require,module,exports){ -(function (process){ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -// resolves . and .. elements in a path array with directory names there -// must be no slashes, empty elements, or device names (c:\) in the array -// (so also no leading and trailing slashes - it does not distinguish -// relative and absolute paths) -function normalizeArray(parts, allowAboveRoot) { - // if the path tries to go above the root, `up` ends up > 0 - var up = 0; - for (var i = parts.length - 1; i >= 0; i--) { - var last = parts[i]; - if (last === '.') { - parts.splice(i, 1); - } else if (last === '..') { - parts.splice(i, 1); - up++; - } else if (up) { - parts.splice(i, 1); - up--; - } - } - - // if the path is allowed to go above the root, restore leading ..s - if (allowAboveRoot) { - for (; up--; up) { - parts.unshift('..'); - } - } - - return parts; -} - -// Split a filename into [root, dir, basename, ext], unix version -// 'root' is just a slash, or nothing. -var splitPathRe = - /^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/; -var splitPath = function(filename) { - return splitPathRe.exec(filename).slice(1); -}; - -// path.resolve([from ...], to) -// posix version -exports.resolve = function() { - var resolvedPath = '', - resolvedAbsolute = false; - - for (var i = arguments.length - 1; i >= -1 && !resolvedAbsolute; i--) { - var path = (i >= 0) ? arguments[i] : process.cwd(); - - // Skip empty and invalid entries - if (typeof path !== 'string') { - throw new TypeError('Arguments to path.resolve must be strings'); - } else if (!path) { - continue; - } - - resolvedPath = path + '/' + resolvedPath; - resolvedAbsolute = path.charAt(0) === '/'; - } - - // At this point the path should be resolved to a full absolute path, but - // handle relative paths to be safe (might happen when process.cwd() fails) - - // Normalize the path - resolvedPath = normalizeArray(filter(resolvedPath.split('/'), function(p) { - return !!p; - }), !resolvedAbsolute).join('/'); - - return ((resolvedAbsolute ? '/' : '') + resolvedPath) || '.'; -}; - -// path.normalize(path) -// posix version -exports.normalize = function(path) { - var isAbsolute = exports.isAbsolute(path), - trailingSlash = substr(path, -1) === '/'; - - // Normalize the path - path = normalizeArray(filter(path.split('/'), function(p) { - return !!p; - }), !isAbsolute).join('/'); - - if (!path && !isAbsolute) { - path = '.'; - } - if (path && trailingSlash) { - path += '/'; - } - - return (isAbsolute ? '/' : '') + path; -}; - -// posix version -exports.isAbsolute = function(path) { - return path.charAt(0) === '/'; -}; - -// posix version -exports.join = function() { - var paths = Array.prototype.slice.call(arguments, 0); - return exports.normalize(filter(paths, function(p, index) { - if (typeof p !== 'string') { - throw new TypeError('Arguments to path.join must be strings'); - } - return p; - }).join('/')); -}; - - -// path.relative(from, to) -// posix version -exports.relative = function(from, to) { - from = exports.resolve(from).substr(1); - to = exports.resolve(to).substr(1); - - function trim(arr) { - var start = 0; - for (; start < arr.length; start++) { - if (arr[start] !== '') break; - } - - var end = arr.length - 1; - for (; end >= 0; end--) { - if (arr[end] !== '') break; - } - - if (start > end) return []; - return arr.slice(start, end - start + 1); - } - - var fromParts = trim(from.split('/')); - var toParts = trim(to.split('/')); - - var length = Math.min(fromParts.length, toParts.length); - var samePartsLength = length; - for (var i = 0; i < length; i++) { - if (fromParts[i] !== toParts[i]) { - samePartsLength = i; - break; - } - } - - var outputParts = []; - for (var i = samePartsLength; i < fromParts.length; i++) { - outputParts.push('..'); - } - - outputParts = outputParts.concat(toParts.slice(samePartsLength)); - - return outputParts.join('/'); -}; - -exports.sep = '/'; -exports.delimiter = ':'; - -exports.dirname = function(path) { - var result = splitPath(path), - root = result[0], - dir = result[1]; - - if (!root && !dir) { - // No dirname whatsoever - return '.'; - } - - if (dir) { - // It has a dirname, strip trailing slash - dir = dir.substr(0, dir.length - 1); - } - - return root + dir; -}; - - -exports.basename = function(path, ext) { - var f = splitPath(path)[2]; - // TODO: make this comparison case-insensitive on windows? - if (ext && f.substr(-1 * ext.length) === ext) { - f = f.substr(0, f.length - ext.length); - } - return f; -}; - - -exports.extname = function(path) { - return splitPath(path)[3]; -}; - -function filter (xs, f) { - if (xs.filter) return xs.filter(f); - var res = []; - for (var i = 0; i < xs.length; i++) { - if (f(xs[i], i, xs)) res.push(xs[i]); - } - return res; -} - -// String.prototype.substr - negative index don't work in IE8 -var substr = 'ab'.substr(-1) === 'b' - ? function (str, start, len) { return str.substr(start, len) } - : function (str, start, len) { - if (start < 0) start = str.length + start; - return str.substr(start, len); - } -; - -}).call(this,require('_process')) -},{"_process":4}],4:[function(require,module,exports){ // shim for using process in browser var process = module.exports = {}; @@ -6804,7 +6593,86 @@ process.chdir = function (dir) { }; process.umask = function() { return 0; }; -},{}],5:[function(require,module,exports){ +},{}],4:[function(require,module,exports){ +(function (setImmediate,clearImmediate){(function (){ +var nextTick = require('process/browser.js').nextTick; +var apply = Function.prototype.apply; +var slice = Array.prototype.slice; +var immediateIds = {}; +var nextImmediateId = 0; + +// DOM APIs, for completeness + +exports.setTimeout = function() { + return new Timeout(apply.call(setTimeout, window, arguments), clearTimeout); +}; +exports.setInterval = function() { + return new Timeout(apply.call(setInterval, window, arguments), clearInterval); +}; +exports.clearTimeout = +exports.clearInterval = function(timeout) { timeout.close(); }; + +function Timeout(id, clearFn) { + this._id = id; + this._clearFn = clearFn; +} +Timeout.prototype.unref = Timeout.prototype.ref = function() {}; +Timeout.prototype.close = function() { + this._clearFn.call(window, this._id); +}; + +// Does not start the time, just sets up the members needed. +exports.enroll = function(item, msecs) { + clearTimeout(item._idleTimeoutId); + item._idleTimeout = msecs; +}; + +exports.unenroll = function(item) { + clearTimeout(item._idleTimeoutId); + item._idleTimeout = -1; +}; + +exports._unrefActive = exports.active = function(item) { + clearTimeout(item._idleTimeoutId); + + var msecs = item._idleTimeout; + if (msecs >= 0) { + item._idleTimeoutId = setTimeout(function onTimeout() { + if (item._onTimeout) + item._onTimeout(); + }, msecs); + } +}; + +// That's not how node.js implements it but the exposed api is the same. +exports.setImmediate = typeof setImmediate === "function" ? setImmediate : function(fn) { + var id = nextImmediateId++; + var args = arguments.length < 2 ? false : slice.call(arguments, 1); + + immediateIds[id] = true; + + nextTick(function onNextTick() { + if (immediateIds[id]) { + // fn.call() is faster so we optimize for the common use-case + // @see http://jsperf.com/call-apply-segu + if (args) { + fn.apply(null, args); + } else { + fn.call(null); + } + // Prevent ids from leaking + exports.clearImmediate(id); + } + }); + + return id; +}; + +exports.clearImmediate = typeof clearImmediate === "function" ? clearImmediate : function(id) { + delete immediateIds[id]; +}; +}).call(this)}).call(this,require("timers").setImmediate,require("timers").clearImmediate) +},{"process/browser.js":3,"timers":4}],5:[function(require,module,exports){ /** @license zlib.js 2012 - imaya [ https://github.com/imaya/zlib.js ] The MIT License */(function() {'use strict';function n(e){throw e;}var p=void 0,aa=this;function t(e,b){var d=e.split("."),c=aa;!(d[0]in c)&&c.execScript&&c.execScript("var "+d[0]);for(var a;d.length&&(a=d.shift());)!d.length&&b!==p?c[a]=b:c=c[a]?c[a]:c[a]={}};var x="undefined"!==typeof Uint8Array&&"undefined"!==typeof Uint16Array&&"undefined"!==typeof Uint32Array&&"undefined"!==typeof DataView;new (x?Uint8Array:Array)(256);var y;for(y=0;256>y;++y)for(var A=y,ba=7,A=A>>>1;A;A>>>=1)--ba;function B(e,b,d){var c,a="number"===typeof b?b:b=0,f="number"===typeof d?d:e.length;c=-1;for(a=f&7;a--;++b)c=c>>>8^C[(c^e[b])&255];for(a=f>>3;a--;b+=8)c=c>>>8^C[(c^e[b])&255],c=c>>>8^C[(c^e[b+1])&255],c=c>>>8^C[(c^e[b+2])&255],c=c>>>8^C[(c^e[b+3])&255],c=c>>>8^C[(c^e[b+4])&255],c=c>>>8^C[(c^e[b+5])&255],c=c>>>8^C[(c^e[b+6])&255],c=c>>>8^C[(c^e[b+7])&255];return(c^4294967295)>>>0} var D=[0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125561021,3887607047,2428444049,498536548,1789927666,4089016648,2227061214,450548861,1843258603,4107580753,2211677639,325883990,1684777152,4251122042,2321926636,335633487,1661365465,4195302755,2366115317,997073096,1281953886,3579855332,2724688242,1006888145,1258607687,3524101629,2768942443,901097722,1119000684,3686517206,2898065728,853044451,1172266101,3705015759, 2882616665,651767980,1373503546,3369554304,3218104598,565507253,1454621731,3485111705,3099436303,671266974,1594198024,3322730930,2970347812,795835527,1483230225,3244367275,3060149565,1994146192,31158534,2563907772,4023717930,1907459465,112637215,2680153253,3904427059,2013776290,251722036,2517215374,3775830040,2137656763,141376813,2439277719,3865271297,1802195444,476864866,2238001368,4066508878,1812370925,453092731,2181625025,4111451223,1706088902,314042704,2344532202,4240017532,1658658271,366619977, @@ -6984,7 +6852,7 @@ module.exports = Tokenizer; "use strict"; var Tokenizer = require("./Tokenizer"); -var DictionaryLoader = require("./loader/NodeDictionaryLoader"); +var BrowserDictionaryLoader = require("./loader/BrowserDictionaryLoader"); /** * TokenizerBuilder create Tokenizer instance. @@ -7005,7 +6873,7 @@ function TokenizerBuilder(option) { * @param {TokenizerBuilder~onLoad} callback Callback function */ TokenizerBuilder.prototype.build = function (callback) { - var loader = new DictionaryLoader(this.dic_path); + var loader = new BrowserDictionaryLoader(this.dic_path); loader.load(function (err, dic) { callback(err, new Tokenizer(dic)); }); @@ -7020,7 +6888,7 @@ TokenizerBuilder.prototype.build = function (callback) { module.exports = TokenizerBuilder; -},{"./Tokenizer":6,"./loader/NodeDictionaryLoader":19}],8:[function(require,module,exports){ +},{"./Tokenizer":6,"./loader/BrowserDictionaryLoader":19}],8:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors @@ -8163,7 +8031,6 @@ module.exports = BrowserDictionaryLoader; "use strict"; -var path = require("path"); var async = require("async"); var DynamicDictionaries = require("../dict/DynamicDictionaries"); @@ -8194,7 +8061,7 @@ DictionaryLoader.prototype.load = function (load_callback) { // Trie function (callback) { async.map([ "base.dat.gz", "check.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -8214,7 +8081,7 @@ DictionaryLoader.prototype.load = function (load_callback) { // Token info dictionaries function (callback) { async.map([ "tid.dat.gz", "tid_pos.dat.gz", "tid_map.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -8234,7 +8101,7 @@ DictionaryLoader.prototype.load = function (load_callback) { }, // Connection cost matrix function (callback) { - loadArrayBuffer(path.join(dic_path, "cc.dat.gz"), function (err, buffer) { + loadArrayBuffer(dic_path + "cc.dat.gz", function (err, buffer) { if(err) { return callback(err); } @@ -8246,7 +8113,7 @@ DictionaryLoader.prototype.load = function (load_callback) { // Unknown dictionaries function (callback) { async.map([ "unk.dat.gz", "unk_pos.dat.gz", "unk_map.dat.gz", "unk_char.dat.gz", "unk_compat.dat.gz", "unk_invoke.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -8282,7 +8149,7 @@ DictionaryLoader.prototype.load = function (load_callback) { module.exports = DictionaryLoader; -},{"../dict/DynamicDictionaries":11,"async":1,"path":3}],21:[function(require,module,exports){ +},{"../dict/DynamicDictionaries":11,"async":1}],21:[function(require,module,exports){ /* * Copyright 2014 Takuya Asano * Copyright 2010-2014 Atilika Inc. and contributors diff --git a/src/TokenizerBuilder.js b/src/TokenizerBuilder.js index 9ef5c6a2efc63e8b12735a8a9f1cb08d6c52c20c..98881e9fd731047c3fca848a71ede7e381e74f51 100644 --- a/src/TokenizerBuilder.js +++ b/src/TokenizerBuilder.js @@ -18,7 +18,7 @@ "use strict"; var Tokenizer = require("./Tokenizer"); -var DictionaryLoader = require("./loader/NodeDictionaryLoader"); +var BrowserDictionaryLoader = require("./loader/BrowserDictionaryLoader"); /** * TokenizerBuilder create Tokenizer instance. @@ -39,7 +39,7 @@ function TokenizerBuilder(option) { * @param {TokenizerBuilder~onLoad} callback Callback function */ TokenizerBuilder.prototype.build = function (callback) { - var loader = new DictionaryLoader(this.dic_path); + var loader = new BrowserDictionaryLoader(this.dic_path); loader.load(function (err, dic) { callback(err, new Tokenizer(dic)); }); diff --git a/src/loader/DictionaryLoader.js b/src/loader/DictionaryLoader.js index 5f88c0b7f9a786dd8c072a7b84ae86a6f31412cb..3d6f8a67e16d251b3e4ba4dbbbc947679c364382 100644 --- a/src/loader/DictionaryLoader.js +++ b/src/loader/DictionaryLoader.js @@ -17,7 +17,6 @@ "use strict"; -var path = require("path"); var async = require("async"); var DynamicDictionaries = require("../dict/DynamicDictionaries"); @@ -48,7 +47,7 @@ DictionaryLoader.prototype.load = function (load_callback) { // Trie function (callback) { async.map([ "base.dat.gz", "check.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -68,7 +67,7 @@ DictionaryLoader.prototype.load = function (load_callback) { // Token info dictionaries function (callback) { async.map([ "tid.dat.gz", "tid_pos.dat.gz", "tid_map.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } @@ -88,7 +87,7 @@ DictionaryLoader.prototype.load = function (load_callback) { }, // Connection cost matrix function (callback) { - loadArrayBuffer(path.join(dic_path, "cc.dat.gz"), function (err, buffer) { + loadArrayBuffer(dic_path + "cc.dat.gz", function (err, buffer) { if(err) { return callback(err); } @@ -100,7 +99,7 @@ DictionaryLoader.prototype.load = function (load_callback) { // Unknown dictionaries function (callback) { async.map([ "unk.dat.gz", "unk_pos.dat.gz", "unk_map.dat.gz", "unk_char.dat.gz", "unk_compat.dat.gz", "unk_invoke.dat.gz" ], function (filename, _callback) { - loadArrayBuffer(path.join(dic_path, filename), function (err, buffer) { + loadArrayBuffer(dic_path + filename, function (err, buffer) { if(err) { return _callback(err); } diff --git a/src/loader/NodeDictionaryLoader.js b/src/loader/NodeDictionaryLoader.js deleted file mode 100644 index 26eb79249121efe39bd5ae77c17e1caa197fb4ce..0000000000000000000000000000000000000000