import * as urlLib from 'url';

import * as pluginUtils from './loader/utils.js';

import * as utils from './utils.js';

import log from '../logging.js';

import * as oembedUtils from './oembed.js';

import * as pluginLoader from './loader/pluginLoader.js';

import requestWrapper from './request.js';

import { cache } from './cache.js';

import * as htmlUtils from './html-utils.js';

import * as metaUtils from './plugins/system/meta/utils.js';

import mediaPlugin from './plugins/validators/media.js';

import { difference, intersection } from '../utils.js';

const plugins = pluginLoader._plugins,

pluginsModules = pluginLoader._pluginsModules,

providedParamsDict = pluginLoader._providedParamsDict,

pluginsList = pluginLoader._pluginsList,

usedParamsDict = pluginLoader._usedParamsDict,

postPluginsList = pluginLoader._postPluginsList,

templates = pluginLoader._templates,

PLUGIN_METHODS = pluginUtils.PLUGIN_METHODS;

const utilsModules = {

utils: utils,

htmlUtils: htmlUtils,

metaUtils: metaUtils,

plugins: pluginsModules,

prepareLinkMedia: mediaPlugin.prepareLink,

log: log,

cache: cache

};

/*

function findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams) {

/*

// Do not process plugin twice (if recursive dependency).

if (pluginId in scannedPluginsIds) {

return;

}

var plugin = plugins[pluginId];

if (!plugin) {

return;

}

// Register result array.

scannedPluginsIds[plugin.id] = true;

// Process all mixins.

if (!mandatoryParams) {

var mixins = plugin.module.mixins;

if (mixins) {

for(var i = 0; i < mixins.length; i++) {

findPluginMethods(mixins[i], loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains);

}

}

}

var usedPluginMethods = usedMethods[plugin.id] = usedMethods[plugin.id] || {};

// Check each plugin method.

for(var i = 0; i < PLUGIN_METHODS.length; i++) {

var method = PLUGIN_METHODS[i];

if (method in plugin.methods) {

// Skip used method.

if (usedPluginMethods && (method in usedPluginMethods)) {

continue;

}

var params = plugin.methods[method];

// If mandatory params mode.

if (mandatoryParams && mandatoryParams.length > 0) {

if (intersection(params, mandatoryParams).length === 0) {

// Skip method if its not using mandatory params.

continue;

}

}

var absentParams = difference(params, loadedParams);

// If "__" (as in "__statusCode") or "...Error" (as in "oembedError")

// super mandatory params are absent - skip the plugin.

var hasMandatoryParams = false;

for(var j = 0; j < absentParams.length && !hasMandatoryParams; j++) {

var param = absentParams[j];

if (param.substr(0, 2) === "__" || /error$/i.test(param)) {

hasMandatoryParams = true;

}

}

if (hasMandatoryParams) {

continue;

}

// 'ulrMatch' workaround.

// Each plugin can have own match, or no match at all.

var urlMatchParamIdx = absentParams.indexOf('urlMatch');

if (urlMatchParamIdx > -1) {

if (pluginsUrlMatches[pluginId]) {

// If 'urlMatch' for plugin found - remove from 'absentParams'.

absentParams.splice(urlMatchParamIdx, 1);

} else {

// Skip method with 'urlMatch' required. Match not found for that plugin.

continue;

}

}

if (absentParams.length > 0) {

// This branch finds methods in upper dependencies tree.

var pluginsId = findPluginsForAbsentParams(absentParams, usedDomains, usedParams);

// Find absent params in other plugins 'provides' attribute.

for(var j = 0; j < pluginsId.length; j++) {

var foundPluginId = pluginsId[j];

findPluginMethods(foundPluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains);

}

} else {

// This branch goes down by the tree.

// Mark method used but not finished.

usedPluginMethods[method] = 1;

// Method will run, store in result set.

methods.push({

pluginId: pluginId,

name: method,

handle: plugin.module[method]

});

}

}

}

}

/*

function runMethods(methods, context, pluginsUrlMatches, options, asyncMethodCb) {

// Sync results list.

var results = [];

for(var i = 0; i < methods.length; i++) {

(function() {

var method = methods[i];

var plugin = plugins[method.pluginId];

var params = plugin.methods[method.name];

var args = [];

if (options.debug) {

var methodTimer = utils.createTimer();

}

var asyncMethod = false;

var timeout;

// Result callback for both sync and async methods.

var callback = function(error, data) {

if (timeout) {

clearTimeout(timeout);

timeout = null;

} else if (asyncMethod) {

return;

}

if (data) {

for(var key in data) {

var v = data[key];

if (v === null || (typeof v === 'undefined') || (typeof v === 'number' && isNaN(v))) {

delete data[key];

}

}

}

var result = {

method: method,

data: data

};

if (options.debug) {

result.time = methodTimer();

}

if (error) {

if (error.code) {

var statusCode = error.code;

error = {};

error[SYS_ERRORS.responseStatusCode] = statusCode;

}

if (error instanceof Error) {

error = error.toString();

}

result.error = error;

}

// TODO: rerun plugins check.

if (asyncMethod) {

// Grant real async if callback was called before function return.

process.nextTick(function() {

asyncMethodCb(null, [result]);

});

} else {

// Mark method finished - later, on results callback.

results.push(result);

}

};

// Prepare specific params (cb, urlMatch).

for(var j = 0; j < params.length; j++) {

var param = params[j];

if (param === 'cb') {

// Generate callback param.

asyncMethod = true;

args.push(callback);

continue;

} else if (param === 'urlMatch') {

// Generate 'urlMatch' param depending on plugin.

args.push(pluginsUrlMatches[method.pluginId]);

continue;

}

args.push(context[param]);

}

try {

// TODO: check timeout for async method.

if (asyncMethod) {

timeout = setTimeout(function() {

// In case of async timeout - call 'callback' with error param.

callback(SYS_ERRORS.timeout);

}, options.timeout || CONFIG.RESPONSE_TIMEOUT);

}

// Call method.

var result = method.handle.apply(plugin.module, args);

} catch(ex) {

// Immediately stop method with error.

callback(ex);

if (!asyncMethod) {

// Prevent run sync callback.

asyncMethod = true;

}

}

// Sync callback.

if (!asyncMethod) {

callback(null, result);

} else if (result) {

console.error(" -- Async plugin returned data, but async call of `cb` expected:", method.pluginId + "." + method.name);

}

})();

}

// Call sync methods result in next tick (imitate real async).

if (results.length) {

process.nextTick(function() {

asyncMethodCb(null, results);

});

}

}

/*

function runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb) {

var loadedParams = Object.keys(context);

// var mandatoryParams = difference(loadedParams, Object.keys(usedParams));

// Reset scanned plugins for each iteration.

var scannedPluginsIds = {};

// Methods to run will be here.

var methods = [];

// Find methods in each required plugin.

for(var i = 0; i < requiredPlugins.length; i++) {

var plugin = requiredPlugins[i];

findPluginMethods(plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains);

}

// If has new unused params (mandatoryParams) - then find plugins which can use them.

// if (mandatoryParams && mandatoryParams.length > 0) {

// var secondaryPlugins = findPluginsForMandatoryParams(mandatoryParams, usedDomains);

// // Find methods in plugins, which can use mandatory params.

// for(var i = 0; i < secondaryPlugins.length; i++) {

// var pluginId = secondaryPlugins[i];

// findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams);

// }

// }

// Run found methods.

runMethods(methods, context, pluginsUrlMatches, options, asyncMethodCb);

return methods.length;

}

function getPluginsSet(uri, options, usedParams) {

var initialPlugins = [],

usedDomains,

isDomainPluginsMode = false,

pluginsUrlMatches = {};

if (options.fetchParam) {

var paramPlugins = providedParamsDict[options.fetchParam];

if (paramPlugins && paramPlugins.length) {

// Store dependency param. Need to determine mandatory params in feature.

usedParams[options.fetchParam] = true;

for(var k = 0; k < paramPlugins.length; k++) {

var foundPluginId = paramPlugins[k];

initialPlugins.push(plugins[foundPluginId]);

}

}

} else {

const domain = pluginLoader.getDomainForPlugin(uri);

var pluginMatchesByDomains = {};

function registerDomainPlugin(plugin, match) {

usedDomains = usedDomains || {};

usedDomains[plugin.domain] = true;

var domainPlugins = pluginMatchesByDomains[plugin.domain] = pluginMatchesByDomains[plugin.domain] || {};

domainPlugins[plugin.id] = !!match;

if (plugin.mixinAllGeneric) {

options.mixAllWithDomainPlugin = true;

}

}

if (!options.useOnlyGenericPlugins) {

for(var i = 0; i < pluginsList.length; i++) {

var plugin = pluginsList[i];

var match = plugin.pluginMatchesUrl(domain, uri);

if (match) {

if (match === true) {

// Match without regexp.

registerDomainPlugin(plugin, null);

} else {

registerDomainPlugin(plugin, match);

pluginsUrlMatches[plugin.id] = match;

}

}

}

}

function addAllGeneric() {

// Use all generic plugins.

for(var i = 0; i < pluginsList.length; i++) {

var plugin = pluginsList[i];

if (!plugin.domain && !plugin.custom) {

var match = plugin.pluginReMatchesUrl(domain, uri);

if (match) {

pluginsUrlMatches[plugin.id] = match;

}

// false - has re, no match.

if (match !== false) {

initialPlugins.push(plugin);

}

}

}

}

// In domain debug: add all plugins before domain plugins to make them low priority.

if (options.mixAllWithDomainPlugin) {

addAllGeneric();

}

for(var d in pluginMatchesByDomains) {

var domainPlugins = pluginMatchesByDomains[d];

var matchedPluginsNames = [];

// Find domain plugins with re match.

for(var pluginId in domainPlugins) {

var match = domainPlugins[pluginId];

if (match) {

matchedPluginsNames.push(pluginId);

}

}

// Add all domain plugins without re.

// Old: If no re match - use only non-re domain plugins.

//if (matchedPluginsNames.length === 0) {

for(var pluginId in domainPlugins) {

var plugin = plugins[pluginId];

if (!plugin.re.length) {

matchedPluginsNames.push(pluginId);

}

}

//}

for(var i = 0; i < matchedPluginsNames.length; i++) {

var pluginId = matchedPluginsNames[i];

isDomainPluginsMode = true;

initialPlugins.push(plugins[pluginId]);

}

}

// If not domain or no domain plugins - fill with generic.

if (initialPlugins.length === 0) {

addAllGeneric();

}

if (options.forceParams) {

// TODO: replace forEach

options.forceParams.forEach(function(param) {

var paramPlugins = providedParamsDict[param];

if (paramPlugins && paramPlugins.length) {

// Store dependency param. Need to determine mandatory params in feature.

usedParams[param] = true;

for(var k = 0; k < paramPlugins.length; k++) {

var foundPluginId = paramPlugins[k];

var exists = initialPlugins.find(function(plugin) {

return plugin.id === foundPluginId;

});

if (!exists) {

initialPlugins.push(plugins[foundPluginId]);

}

}

}

});

}

}

return {

initialPlugins: initialPlugins,

pluginsUrlMatches: pluginsUrlMatches,

usedDomains: usedDomains,

isDomainPluginsMode: isDomainPluginsMode

};

}

/*

function findPluginsForMandatoryParams(mandatoryParams, usedDomains) {

var foundPluginsDict = {},

result = [];

// TODO: gether secondaryPlugins by dict, who uses mandatory params. Not full for-loop.

for(var i = 0; i < mandatoryParams.length; i++) {

var pluginsIds = usedParamsDict[mandatoryParams[i]];

if (pluginsIds) {

for(var j = 0; j < pluginsIds.length; j++) {

var pluginId = pluginsIds[j];

var plugin = plugins[pluginId];

// Prevent duplicates.

// Find only generic plugins or plugins from matched domain.

// Skip custom plugins.

if (!(pluginId in foundPluginsDict) && !plugin.custom && (!plugin.domain || (usedDomains && (plugin.domain in usedDomains)))) {

foundPluginsDict[pluginId] = true;

result.push(pluginId);

}

}

}

}

return result;

}

/*

function findPluginsForAbsentParams(absentParams, usedDomains, usedParams) {

var foundPluginsDict = {},

result = [];

for(var i = 0; i < absentParams.length; i++) {

var param = absentParams[i];

// Find absent params in other plugins 'provides' attribute.

var paramPlugins = providedParamsDict[param];

if (paramPlugins && paramPlugins.length) {

for (var j = 0; j < paramPlugins.length; j++) {

var pluginId = paramPlugins[j];

var plugin = plugins[pluginId];

// Prevent duplicates.

// Find only generic plugins or plugins from matched domain.

if (!(pluginId in foundPluginsDict) && (!plugin.domain || (usedDomains && (plugin.domain in usedDomains)))) {

usedParams[param] = true;

foundPluginsDict[pluginId] = true;

result.push(pluginId);

}

}

}

}

return result;

}

function runPostPluginsIterationCall(methodName, iterationPluginContexts) {

for(var i = 0; i < postPluginsList.length; i++) {

(function() {

var plugin = postPluginsList[i];

var handle = plugin.module[methodName];

if (!handle) {

return;

}

var iterationPluginContext = iterationPluginContexts[plugin.id] = iterationPluginContexts[plugin.id] || {};

try {

handle.apply(plugin.module, [iterationPluginContext]);

} catch(ex) {

console.error(" -- Post plugin error in", plugin.id + "." + methodName, ex);

}

})();

}

}

function runPostPlugins(link, dataRecord, usedMethods, context, pluginsContexts, iterationPluginContexts, options, asyncMethodCb) {

// Sync results list.

var results = [];

for(var i = 0; i < postPluginsList.length; i++) {

(function() {

// This will prevent lower priority plugins if previous _sync_ plugin placed link.error.

if (link.error) {

return;

}

var plugin = postPluginsList[i];

var method = {

pluginId: plugin.id,

name: 'prepareLink',

handle: plugin.module['prepareLink']

};

var params = plugin.methods[method.name];

var args = [];

var asyncMethod = false;

var timeout;

// Result callback for both sync and async methods.

var callback = function(error, data) {

if (timeout) {

clearTimeout(timeout);

timeout = null;

} else if (asyncMethod) {

return;

}

var result = {

method: method,

data: data

};

if (error) {

if (error instanceof Error) {

error = error.stack;

}

console.error(" -- Post plugin error in", plugin.id + "." + method.name, error);

}

if (asyncMethod) {

// Grant real async if callback was called before function return.

process.nextTick(function() {

// Call 'asyncMethodCb' only for async post plugin, because core must wait it.

asyncMethodCb(null, [result]);

});

} else {

// Mark method finished - later, on results callback.

results.push(result);

}

};

// Prepare specific params (cb, urlMatch).

for(var j = 0; j < params.length; j++) {

var param = params[j];

if (param === 'cb') {

// Generate callback param.

asyncMethod = true;

args.push(callback);

continue;

} else if (param === 'link') {

// Generate 'link' param.

args.push(link);

continue;

} else if (param === 'pluginContext') {

var pluginContext = pluginsContexts[plugin.id] = pluginsContexts[plugin.id] || {};

// Generate 'pluginContext' param.

args.push(pluginContext);

continue;

} else if (param === 'iterationPluginContext') {

var iterationPluginContext = iterationPluginContexts[plugin.id] = iterationPluginContexts[plugin.id] || {};

// Generate 'pluginContext' param.

args.push(iterationPluginContext);

continue;

} else if (param === 'plugin') {

args.push(plugins[dataRecord.method.pluginId]);

continue;

} else if (param === 'templates') {

args.push(templates);

continue;

}

args.push(context[param]);

}

var usedPluginMethods = usedMethods[plugin.id] = usedMethods[plugin.id] || {};

usedPluginMethods[method.name] = (usedPluginMethods[method.name] || 0) + 1;

try {

// TODO: check timeout for async method.

if (asyncMethod) {

timeout = setTimeout(function() {

// In case of async timeout - call 'callback' with error param.

callback(SYS_ERRORS.timeout);

}, options.timeout || CONFIG.RESPONSE_TIMEOUT);

}

// Call method.

var result = method.handle.apply(plugin.module, args);

} catch(ex) {

// Immediately stop method with error.

callback(ex);

if (!asyncMethod) {

// Prevent run sync callback again.

asyncMethod = true;

}

}

// Sync callback.

if (!asyncMethod) {

callback(null, result);

}

})();

}

// Call sync methods result in next tick (imitate real async).

if (results.length) {

process.nextTick(function() {

asyncMethodCb(null, results);

});

}

}

function getResultErrorMessages(result, allResults) {

if (!result) {

return false;

}

for(var i = 0; i < result.length; i++) {

var r = result[i];

if (r.error) {

// Get message from plugin error.

useMessage(allResults, r.error);

}

}

}

function useResult(usedMethods, context, pluginsContexts, allResults, result, options, asyncMethodCb) {

if (!result) {

return false;

}

var hasNewData = false;

for(var i = 0; i < result.length; i++) {

var r = result[i];

// Mark method finished.

// Mark synced to asyncMethodCb call.

var method = r.method;

var usedPluginMethods = usedMethods[method.pluginId];

usedPluginMethods[method.name] = usedPluginMethods[method.name] - 1;

if (r.error && options.debug) {

console.error(" -- Plugin error", method.pluginId, method.name, r.error);

}

// Collect total result.

allResults.allData.push(r);

if (r.data && r.data.title && !context.title) {

// Store title.

context.title = r.data.title;

}

// Merge data to context.

if (r.data) {

if (!(r.data instanceof Object)) {

// Check if method result is Object.

console.error('Non object returned in', r.method.pluginId, r.method.name);

r.error = 'Non object returned';

} else if (r.method.name === 'getData') {

// Extend context with 'getData' result.

for(var key in r.data) {

if (key === 'message') {

// Get message from plugin data.

useMessage(allResults, r.data);

} else {

if (key === 'htmlparser') {

// Debug http2 state.

allResults.h2 = r.data.htmlparser.h2;

}

// First data has priority, do not override it.

// whitelistRecord - exception for oembed plugin.

if (!(key in context) || key === 'whitelistRecord') {

context[key] = r.data[key];

hasNewData = true;

}

}

}

} else if (r.method.name === "getMeta") {

// Extend unified meta.

for(var key in r.data) {

var v = r.data[key];

// TODO: postprocessing meta plugins.

if (key === 'date') {

v = utils.unifyDate(v);

if (!v) {

// Disable invalid date.

r.data[key] = v;

}

}

if (key === 'duration') {

v = utils.unifyDuration(v);

if (!v) {

// Disable invalid duration.

r.data[key] = v;

}

}

if (key === 'title' || key === 'canonical') {

if (v instanceof Array) {

v = v[0];

}

}

if (key === 'author' && v.match && v.match(/^https?:\/\//)) {

key = 'author_url';

}

if (key === 'author' && /^admin|user/i.test(v)) {

v = '';

}

if (typeof v === 'string' && /^{{.+}}$/.test(v)) {

v = ''; // template wasn't pre-rendered, ignore

}

if (v !== '' && v !== null && ((typeof v === 'string' && !/^\s+$/.test(v)) || typeof v === 'number')) {

// Check meta plugins order.

allResults.meta._sources = allResults.meta._sources || {};

var prevOrder = null, nextOrder = null, pluginId = allResults.meta._sources[key];

if (pluginId && plugins[pluginId] && plugins[r.method.pluginId]) {

prevOrder = plugins[pluginId].order;

nextOrder = plugins[r.method.pluginId].order;

}

if (!prevOrder || !nextOrder || prevOrder < nextOrder) {

allResults.meta[key] = v;

allResults.meta._sources[key] = r.method.pluginId;

}

}

}

} else if (r.method.name === "getVars") {

for(var key in r.data) {

var v = r.data[key];

if (v !== '' && v !== null && ((typeof v === 'string' && !/^\s+$/.test(v)) || typeof v === 'number' || typeof v === 'object')) {