tags, and remove inline fonts
document.body.innerHTML = document.body.innerHTML.replace(/ '); // first item
firstTime = false;
}
else
{
if (text.indexOf('\n\n') == text.lastIndexOf('\n\n'))
text = text.replace('\n\n', '
'); // every item in between
}
}
content.innerHTML = text.replace(/={10,}/g, "====================");
paragraphs = content.getElementsByTagName("P");
var preElements = [];
for (var j = 0; j < paragraphs.length; j++)
{
p = paragraphs[j];
breaks = p.getElementsByTagName("BR");
if (p.innerHTML.indexOf("\t") == -1 && p.innerHTML.indexOf(" ") == -1 && breaks.length >= 1)
{
p.innerHTML = p.innerHTML.replace(/ [ \r\n\s]* /gi, " ");
contentBlocks.push(content);
}
*/
// wow.. talk about a bad site, no paragraphs found so we'll attempt to
// parse content from div's and set our malformedContent flag
if (paragraphs.length == 0)
{
paragraphs = document.getElementsByTagName("DIV");
malformedContent = true;
}
for (var i = 0; i < paragraphs.length; i++)
{
var parentNode = paragraphs[i].parentNode;
// TODO: originally the if/continue statement below checked if the parent
// was the body tag and if it was continued on.. why?
// if the parent happens to be a form element, accessing properties
// such as id or className don't work, or rather it attempts to access
// children so we need to make sure we only deal with string values
if (typeof parentNode.id != "string" || typeof parentNode.className != "string")
continue;
// initialize readability score data
if (typeof parentNode.readability == "undefined")
parentNode.readability = {"contentScore": 0};
parentNode.readability.contentScore = determineContentScore(parentNode.readability.contentScore, parentNode, paragraphs[i]);
// looks like we have possible content candidates, add it
if (parentNode.readability.contentScore > 0)
{
// DEBUG
console.log(parentNode.tagName + " id: " + parentNode.id + " -- class: " + parentNode.className + " -- score: " + parentNode.readability.contentScore);
// careful, only add parent element once!
if (contentBlocks.indexOf(parentNode) == -1)
contentBlocks.push(parentNode);
}
}
/*
// TODO: need to revisit parsing strictly tables/divs content only
if (contentBlocks.length == 0)
{
var paragraphs = document.getElementsByTagName("tbody");
for (var i = 0; i < paragraphs.length; i++)
{
var parentNode = paragraphs[i].parentNode;
// Initialize readability data
if (typeof parentNode.readability == "undefined")
{
parentNode.readability = {"contentScore": determineContentScore(parentNode, paragraphs[i])};
if (parentNode.readability.contentScore > 0)
{
console.log(parentNode.tagName + " id: " + parentNode.id + " -- class: " + parentNode.className + " -- score: " + parentNode.readability.contentScore);
if (contentBlocks.indexOf(parentNode) == -1)
contentBlocks.push(parentNode);
}
}
}
}
*/
removeScripts();
removeStylesheets();
removeStyles();
// DEBUG
console.log("ContentBlocks: " + contentBlocks.length + " -- HighestScore: " + highestScore);
// remove all content elements that aren't of the highest score
var numContentBlocks = contentBlocks.length - 1;
for (var m = numContentBlocks; m >= 0; m--)
{
var contentElement = contentBlocks[m];
// DEBUG
//console.log("id: " + contentElement.id + " -- class: " + contentElement.className + " -- result: " + ((highestScore < 20 && contentElement.readability.contentScore < highestScore) || (contentElement.readability.contentScore < 20)).toString().toUpperCase());
// FIXME: had trouble writing the if/else if as a single if or statement
// FIXME: not sure the minimum score is correct, need to test against wide
// range of content, particularly content divided in 2+ containers
// sometimes our content won't reach such a high score so here we look for an
// acceptable minimum, if our highest score didn't go above twenty remove all
// but the highest
if (highestScore < 20 && contentElement.readability.contentScore < highestScore)
{
contentBlocks.splice(m, 1);
} //otherwise we only remove content blocks that have scored less than that minimum
else if (highestScore > 20 && contentElement.readability.contentScore < 20)
{
contentBlocks.splice(m, 1);
}
}
// with many content containers we need to verify that some
// aren't descendants of others otherwise we'll get multiple output
if (contentBlocks.length > 1)
{
// remove all content elements that are descandants of another
var numContentBlocks = contentBlocks.length - 1;
for (var m = numContentBlocks; m >= 0; m--)
{
var contentElement = contentBlocks[m];
/**
* hasAnyAncestor should work better overall but some sites
* have so many div's up the hierarchy with lots of good keywords
* its hard to keep those out, for those sites
* (http://www.azstarnet.com/news/290815) hasAnyDescendant works
* best so will need to consider changing and QA heavily.
*/
if (hasAnyDescendant(contentElement, contentBlocks))
contentBlocks.splice(m, 1);
}
}
// DEBUG
console.log("ContentBlocks: " + contentBlocks.length);
for (var m = 0; m < contentBlocks.length; m++)
{
var contentElement = contentBlocks[m];
removeElementStyles(contentElement);
// remove any consecutive stuff than stuff
removeNonContentElement(contentElement, "div");
}
//removeNonContentElement(contentElement, "ul");
// clean out anymore possible junk
removeElementByMinWords(contentElement, "form");
removeElementByMinWords(contentElement, "object");
removeElementByMinWords(contentElement, "table", 250);
removeElementByMinWords(contentElement, "h1");
removeElementByMinWords(contentElement, "h2");
removeElementByMinWords(contentElement, "iframe");
articleContent.appendChild(contentElement);
}
// Readability has failed you.. show msg that content was not found
if (contentBlocks.length == 0)
{
articleContent = document.createElement("DIV");
articleContent.innerHTML = 'Sorry, readability was unable to parse this page for content. If you feel like it should have been able to, please let us know by submitting an issue.';
}
return articleContent;
}
//--------------------------------------------------------------------------
//
// ContentParserUtils
//
//--------------------------------------------------------------------------
/**
* Removes any elements of the provided tag name from the specified element
* if it doesn't contain the minimum amount of words.
*
* @param element The element.
* @param tagName The tag name of the elements to be retrieved from within
* the provided element.
* @param minWords The minimum number of words.
*/
function removeElementByMinWords(element, tagName, minWords)
{
// default minimum if none is provided
minWords = minWords || 1000000; // FIXME: not sure why such a higher number!
var elements = element.getElementsByTagName(tagName);
var numElements = elements.length - 1;
for (var i = numElements; i >= 0; i--)
{
var target = elements[i];
// the text content doesn't meet our requirements so remove it
if (getWordCount(target) < minWords)
{
target.parentNode.removeChild(target);
}
}
}
/**
* Removes any instances of the provided non-content element from the
* specified root element if it passes a few tests. First, if a single
* bad keyword is found or second less than 25 words exist within.
*
* @param element The element.
* @param tagName The tag name of the elements to be retrieved from within
* the provided element.
*/
function removeNonContentElement(element, tagName)
{
var elements = element.getElementsByTagName(tagName);
var numElements = elements.length - 1;
// gather counts for other typical elements embedded within and then traverse
// backwards so we can remove elements at the same time without effecting the traversal
for (var i = numElements; i >= 0; i--)
{
var descendant = elements[i];
var p = descendant.getElementsByTagName("p").length;
var img = descendant.getElementsByTagName("img").length;
var li = descendant.getElementsByTagName("li").length;
var a = descendant.getElementsByTagName("a").length;
var embed = descendant.getElementsByTagName("embed").length;
/*
// no basic elements were found at all
if (a == 0 && embed == 0 & img == 0 && li == 0 && p == 0)
{
// retrieve all children to see if it contains any elements
var children = descendant.getElementsByTagName("*");
var containsOnlyText = true;
for (var j = 0; j < children.length; j++)
{
var child = children[j];
// element type found so we don't have an element (e.g. DIV) with just text
if (child.nodeType == 1)
{
containsOnlyText = false;
break;
}
}
//
if (!containsOnlyText)
{
descendant.parentNode.removeChild(descendant);
}
continue;
}
else
{*/
var badKeywords = ["ad", "captcha", "classified", "clear", "comment", "crumbs", "footer", "footnote", "leftcolumn", "listing", "menu", "meta", "module", "nav", "navbar", "rightcolumn", "sidebar", "sponsor", "tab", "tag", "toolbar", "tools", "trackback", "tweetback", "widget"];
// should improve this but for if the element has a single bad keyword remove it
for (var j = 0; j < badKeywords.length; j++)
{
if (descendant.id.toLowerCase().indexOf(badKeywords[j]) >= 0 || descendant.className.toLowerCase().indexOf(badKeywords[j]) >= 0)
{
descendant.parentNode.removeChild(descendant);
descendant = null;
break;
}
}
/*}*/
// found a bad keyword so the element has been removed, continue to the next one
if (!descendant)
continue;
// we have fewer than 25 words.. bad sign..
if (getWordCount(descendant) < 25)
{
// the number of non-paragraph elements is more than actual
// paragraphs or other ominous signs (:) and elements
if (img > p || li > p || a > p || p == 0 || embed > 0)
{
descendant.parentNode.removeChild(descendant);
}
}
}
}
//--------------------------------------------------------------------------
//
// ElementUtils
//
//--------------------------------------------------------------------------
/**
* Returns the text content of the specified element.
*
* @param element The element from which to retrieve its text content.
*
* @return The string content of the specified element.
*/
function getText(element)
{
return (typeof element.textContent != "undefined")
? element.textContent
: element.innerText;
}
/**
* Returns the word count for the specified element.
*
* @param element The element.
*
* @returns A count indicating the number of words
*/
function getWordCount(element)
{
// normalize replaces consecutive spacing with a single space,
// by then triming, we can safely split on a space for a count
return trim(normalize(getText(element))).split(" ").length;
}
/**
* Determines if the specified element has one of the provided array of
* ancestors and if so returns true.
*
* @param element The element.
* @param ancestors An array of possible ancestors.
*
* @returns True if the element has one of the provided ancestors,
* false if it does not.
*/
function hasAnyAncestor(element, ancestors)
{
var parent = element.parentNode;
while (parent != null)
{
// ancestor found!
if (ancestors.indexOf(parent) >= 0)
return true;
parent = parent.parentNode;
}
return false;
}
/**
* Determines if the specified element has one of the provided array of
* descendants and if so returns true.
*
* @param element The element.
* @param descendants An array of possible descendants.
*
* @returns True if the element has one of the provided descendants,
* false if it does not.
*/
function hasAnyDescendant(element, descendants)
{
var elements = element.getElementsByTagName("*");
for (var i = 0; i < elements.length; i++)
{
// descendant found!
if (descendants.indexOf(elements[i]) >= 0)
return true;
}
return false;
}
/**
* Replaces consecutive spaces with a single space.
*/
function normalize(text)
{
return (text || "").replace(/\s{2,}/g, " ");
}
/**
* Replaces consecutive br tags with a single br tag from the specified element.
*
* @param element The element containing consecutive br tags.
*/
function removeBreaks(element)
{
// FIXME: the regex doesn't seem to pick up consecutive br tags, need to revisit
element.innerHTML = element.innerHTML.replace(/((
's into just one
removeBreaks(contentElement);
// this cleanup should only happen if paragraphs were found since
// malformed content suggests div's are used to maintain content
if (!malformedContent)
{
// goes in and removes DIV's that have more non
");
}
/**
* Removes any styles on the specified element.
*
* @param element The element containing the styles to be removed.
*/
function removeElementStyles(element)
{
// bad node, there's not much we can do
if (!element)
return;
// remove any root styles, if we're able
if (typeof element.removeAttribute == "function")
element.removeAttribute("style");
// TODO: do not use firstChild and nextSibling, use childNodes array instead
// prepare to remove styles on all children and siblings
var childElement = element.firstChild;
while (childElement)
{
if (childElement.nodeType == 1)
{
// remove any root styles, if we're able
if (typeof element.removeAttribute == "function")
childElement.removeAttribute("style");
// remove styles recursively
removeElementStyles(childElement);
}
childElement = childElement.nextSibling;
}
}
/**
* Removes all inline or external referencing scripts.
*/
function removeScripts()
{
var scripts = document.getElementsByTagName("SCRIPT");
var numScripts = scripts.length - 1;
for (var i = numScripts; i >= 0; i--)
{
var script = scripts[i];
// remove inline or external referencing scripts (that aren't Readability related)
if (!script.src || (script.src && script.src.indexOf("readability") == -1))
{
script.parentNode.removeChild(script);
}
}
}
/**
* Removes all inline styles.
*/
function removeStyles()
{
var styles = document.getElementsByTagName("STYLE");
var startIndex = styles.length - 1;
for (var i = startIndex; i >= 0; i--)
{
var style = styles[i];
// we prefer to remove the tag completely but if not able we'll clear it
if (style.parentNode)
{
style.parentNode.removeChild(style);
}
else
{
if (style.textContent)
{
style.textContent = "";
}
else
{
// most browsers support textContent but IE has its own way but it
// seems that Firefox supports both, check link for last example
// http://www.phpied.com/the-star-hack-in-ie8-and-dynamic-stylesheets/
// note that if the style tag contains no text content, then
// no styleSheet object is defined either
if (style.styleSheet)
style.styleSheet.cssText = "";
}
}
}
}
/**
* Removes all linked stylesheets.
*/
function removeStylesheets()
{
// TODO: need to do more research, not sure if disabling is enough
// for cross browser compatibility, might consider removal via parent
// just as done in the removeScripts method, but will need to retrieve
// all LINK tags and make sure rel attr is "stylesheet" or that its
// type attr is "text/css"
for (var k = 0; k < document.styleSheets.length; k++)
{
var styleSheet = document.styleSheets[k];
if (styleSheet.href != null && styleSheet.href.lastIndexOf("readability") == -1)
{
styleSheet.disabled = true;
}
}
}
/**
* Removes whitespace from the front and the end of the specified string.
*
* @param text The String whose beginning and ending whitespace will be removed.
*
* @returns A String with whitespace removed from the begining and end
*/
function trim(text)
{
return (text || "").replace(/^\s+|\s+$/g, "");
}