From 10d457ab1a0ae08d8896782b7860ea5e0e872feb Mon Sep 17 00:00:00 2001 From: Dave Winer Date: Wed, 24 Jun 2015 17:49:29 -0400 Subject: [PATCH] v0.61 --- lib/opml.js | 441 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/utils.js | 19 ++- package.json | 35 ++-- pagepark.js | 63 ++++++-- 4 files changed, 528 insertions(+), 30 deletions(-) create mode 100644 lib/opml.js mode change 100755 => 100644 package.json diff --git a/lib/opml.js b/lib/opml.js new file mode 100644 index 0000000..516bcff --- /dev/null +++ b/lib/opml.js @@ -0,0 +1,441 @@ +exports.readOpmlString = readOpmlString; +exports.readOpmlFile = readOpmlFile; +exports.readOpmlUrl = readOpmlUrl; +exports.outlineVisiter = outlineVisiter; + +var request = require ("request"); +var stream = require ("stream"); //6/23/15 by DW +var opmlParser = require ("opmlparser"); //6/23/15 by DW + + +var opmlData = { + flUseOutlineCache: false, + outlineCache: new Object () + } + +function getBoolean (val) { //12/5/13 by DW + switch (typeof (val)) { + case "string": + if (val.toLowerCase () == "true") { + return (true); + } + break; + case "boolean": + return (val); + case "number": + if (val == 1) { + return (true); + } + break; + } + return (false); + } +function getNameAtt (theNode) { + function isAlpha (ch) { + return (((ch >= 'a') && (ch <= 'z')) || ((ch >= 'A') && (ch <= 'Z'))); + } + function isNumeric (ch) { + return ((ch >= '0') && (ch <= '9')); + } + function stripMarkup (s) { //5/24/14 by DW + if ((s === undefined) || (s == null) || (s.length == 0)) { + return (""); + } + return (s.replace (/(<([^>]+)>)/ig, "")); + } + function innerCaseName (text) { //8/12/14 by DW + var s = "", ch, flNextUpper = false; + text = stripMarkup (text); + for (var i = 0; i < text.length; i++) { + ch = text [i]; + if (isAlpha (ch) || isNumeric (ch)) { + if (flNextUpper) { + ch = ch.toUpperCase (); + flNextUpper = false; + } + else { + ch = ch.toLowerCase (); + } + s += ch; + } + else { + if (ch == ' ') { + flNextUpper = true; + } + } + } + return (s); + } + var nameatt = theNode.name; + if (nameatt === undefined) { + nameatt = innerCaseName (theNode.text); + } + return (nameatt); + } +function typeIsDoc (theNode) { + var type = getNodeType (theNode); + return ((type !== undefined) && (type != "include") && (type != "link") && (type != "tweet")); + } +function getNodeType (theNode) { + if (theNode.type == "include") { + return (theNode.includetype); //this allows include nodes to have types + } + else { + return (theNode.type); + } + } +function copyScalars (source, dest) { //8/31/14 by DW + for (var x in source) { + var type, val = source [x]; + if (val instanceof Date) { + val = val.toString (); + } + type = typeof (val); + if ((type != "object") && (type != undefined)) { + dest [x] = val; + } + } + } +function readInclude (theIncludeNode, callback) { + console.log ("readInclude: url == " + theIncludeNode.url); + readOpmlUrl (theIncludeNode.url, function (theOutline, err) { + if (err) { + callback (undefined); + } + else { + expandIncludes (theOutline, function (expandedOutline) { + callback (expandedOutline); + }); + } + }); + } +function outlineVisiter (theOutline, inlevelcallback, outlevelcallback, nodecallback, visitcompletecallback) { + function doLevel (head, path, levelcompletecallback) { + function doOneSub (head, ixsub) { + if ((head.subs !== undefined) && (ixsub < head.subs.length)) { + var sub = head.subs [ixsub], subpath = path + getNameAtt (sub); + if (!getBoolean (sub.iscomment)) { + if ((sub.type == "include") && (!typeIsDoc (sub))) { + nodecallback (sub, subpath); + readInclude (sub, function (theIncludedOutline) { + if (theIncludedOutline !== undefined) { + doLevel (theIncludedOutline, subpath + "/", function () { + outlevelcallback (); + doOneSub (head, ixsub +1); + }); + } + }); + } + else { + if (typeIsDoc (sub)) { + if (sub.type == "index") { + subpath += "/"; + } + nodecallback (sub, subpath); + doOneSub (head, ixsub +1); + } + else { + nodecallback (sub, subpath); + if (sub.subs !== undefined) { + doLevel (sub, subpath + "/", function () { + outlevelcallback (); + doOneSub (head, ixsub +1); + }); + } + else { + doOneSub (head, ixsub +1); + } + } + } + } + else { + doOneSub (head, ixsub +1); + } + } + else { + levelcompletecallback (); + } + } + inlevelcallback (); + if (head.type == "include") { + readInclude (head, function (theIncludedOutline) { + if (theIncludedOutline !== undefined) { + doOneSub (theIncludedOutline, 0); + } + }); + } + else { + doOneSub (head, 0); + } + } + doLevel (theOutline, "", function () { + outlevelcallback (); + visitcompletecallback (); + }); + } +function expandIncludes (theOutline, callback) { + var theNewOutline = new Object (), lastNewNode = theNewOutline, stack = new Array (), currentOutline; + function inlevelcallback () { + stack [stack.length] = currentOutline; + currentOutline = lastNewNode; + if (currentOutline.subs === undefined) { + currentOutline.subs = new Array (); + } + } + function nodecallback (theNode, path) { + var newNode = new Object (); + copyScalars (theNode, newNode); + currentOutline.subs [currentOutline.subs.length] = newNode; + lastNewNode = newNode; + } + function outlevelcallback () { + currentOutline = stack [stack.length - 1]; + stack.length--; //pop the stack + } + outlineVisiter (theOutline, inlevelcallback, outlevelcallback, nodecallback, function () { + callback (theNewOutline); + }); + } +function readOpmlString (s, callback) { + var opmlparser = new opmlParser (); + var outlineArray = new Array (); + var metadata = undefined; + flparseerror = false; + var theStream = new stream.Readable (); + theStream._read = function noop () {}; + theStream.push (s); + theStream.push (null); + theStream.pipe (opmlparser); + + opmlparser.on ("error", function (error) { + console.log ("readOpml: opml parser error == " + error.message); + if (callback != undefined) { + callback (undefined, error); + } + flparseerror = true; + }); + opmlparser.on ("readable", function () { + var outline; + while (outline = this.read ()) { + var ix = Number (outline ["#id"]); + outlineArray [ix] = outline; + if (metadata === undefined) { + metadata = this.meta; + } + } + }); + opmlparser.on ("end", function () { + if (flparseerror) { + return; + } + var theOutline = new Object (); + + //copy elements of the metadata object into the root of the outline + function copyone (name) { + var val = metadata [name]; + if ((val !== undefined) && (val != null)) { + theOutline [name] = val; + } + } + copyone ("title"); + copyone ("datecreated"); + copyone ("datemodified"); + copyone ("ownername"); + copyone ("owneremail"); + copyone ("description"); + + for (var i = 0; i < outlineArray.length; i++) { + var obj = outlineArray [i]; + if (obj != null) { + var idparent = obj ["#parentid"], parent; + if (idparent == 0) { + parent = theOutline; + } + else { + parent = outlineArray [idparent]; + } + if (parent.subs === undefined) { + parent.subs = new Array (); + } + parent.subs [parent.subs.length] = obj; + delete obj ["#id"]; + delete obj ["#parentid"]; + } + } + expandIncludes (theOutline, function (expandedOutline) { + if (callback != undefined) { + callback (expandedOutline, undefined); + } + }); + }); + } +function readOpmlFile (f, callback) { + var outlineArray = new Array (); + var fstream = fs.createReadStream (f); + var opmlparser = new opmlParser (); + var metadata = undefined; + flparseerror = false; + + fstream.pipe (opmlparser); + + opmlparser.on ("error", function (error) { + console.log ("readOpml: opml parser error == " + error.message); + if (callback != undefined) { + callback (undefined, error); + } + flparseerror = true; + }); + opmlparser.on ("readable", function () { + var outline; + while (outline = this.read ()) { + var ix = Number (outline ["#id"]); + outlineArray [ix] = outline; + if (metadata === undefined) { + metadata = this.meta; + } + } + }); + opmlparser.on ("end", function () { + if (flparseerror) { + return; + } + var theOutline = new Object (); + + //copy elements of the metadata object into the root of the outline + function copyone (name) { + var val = metadata [name]; + if ((val !== undefined) && (val != null)) { + theOutline [name] = val; + } + } + copyone ("title"); + copyone ("datecreated"); + copyone ("datemodified"); + copyone ("ownername"); + copyone ("owneremail"); + copyone ("description"); + + for (var i = 0; i < outlineArray.length; i++) { + var obj = outlineArray [i]; + if (obj != null) { + var idparent = obj ["#parentid"], parent; + if (idparent == 0) { + parent = theOutline; + } + else { + parent = outlineArray [idparent]; + } + if (parent.subs === undefined) { + parent.subs = new Array (); + } + parent.subs [parent.subs.length] = obj; + delete obj ["#id"]; + delete obj ["#parentid"]; + } + } + + expandIncludes (theOutline, function (expandedOutline) { + if (callback != undefined) { + callback (expandedOutline, undefined); + } + }); + + }); + } +function readOpmlUrl (urlOutline, callback) { + if (opmlData.flUseOutlineCache && (opmlData.outlineCache [urlOutline] !== undefined)) { + if (callback !== undefined) { + callback (opmlData.outlineCache [urlOutline], undefined); + } + } + else { + var outlineArray = new Array (); + var opmlparser = new opmlParser (); + var metadata = undefined; + var flparseerror = false; + var req; + var theRequest = { + url: urlOutline, + headers: { + "Accept": "text/x-opml, */*", + } + }; + req = request (theRequest); + + req.on ("response", function (res) { + var stream = this; + if (res.statusCode == 200) { + stream.pipe (opmlparser); + } + }); + req.on ("error", function (res) { + console.log ("readOpml: error reading outline. urlOutline == " + urlOutline); + if (callback != undefined) { + callback (undefined, res); + } + }); + opmlparser.on ("error", function (error) { + console.log ("readOpml: opml parser error == " + error.message); + if (callback != undefined) { + callback (undefined, error); + } + flparseerror = true; + }); + opmlparser.on ("readable", function () { + var outline; + while (outline = this.read ()) { + var ix = Number (outline ["#id"]); + outlineArray [ix] = outline; + if (metadata === undefined) { + metadata = this.meta; + } + } + }); + opmlparser.on ("end", function () { + if (flparseerror) { + return; + } + var theOutline = new Object (); + + //copy elements of the metadata object into the root of the outline + function copyone (name) { + var val = metadata [name]; + if ((val !== undefined) && (val != null)) { + theOutline [name] = val; + } + } + copyone ("title"); + copyone ("datecreated"); + copyone ("datemodified"); + copyone ("ownername"); + copyone ("owneremail"); + copyone ("description"); + + for (var i = 0; i < outlineArray.length; i++) { + var obj = outlineArray [i]; + if (obj != null) { + var idparent = obj ["#parentid"], parent; + if (idparent == 0) { + parent = theOutline; + } + else { + parent = outlineArray [idparent]; + } + if (parent.subs === undefined) { + parent.subs = new Array (); + } + parent.subs [parent.subs.length] = obj; + delete obj ["#id"]; + delete obj ["#parentid"]; + } + } + if (opmlData.flUseOutlineCache) { + opmlData.outlineCache [urlOutline] = theOutline; + } + if (callback != undefined) { + callback (theOutline, undefined); + } + }); + } + } + diff --git a/lib/utils.js b/lib/utils.js index 5fbe0a2..85b4d24 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -355,6 +355,22 @@ function random (lower, upper) { function removeMultipleBlanks (s) { //7/30/14 by DW return (s.toString().replace (/ +/g, " ")); } +function jsonStringify (jstruct, flFixBreakage) { //7/30/14 by DW + //Changes + //6/16/15; 10:43:25 AM by DW + //Andrew Shell reported an issue in the encoding of JSON that's solved by doing character replacement. + //However, this is too big a change to make for all the code that calls this library routine, so we added a boolean flag, flFixBreakage. + //If this proves to be harmless, we'll change the default to true. + //http://river4.smallpict.com/2015/06/16/jsonEncodingIssueSolved.html + if (flFixBreakage === undefined) { + flFixBreakage = false; + } + var s = JSON.stringify (jstruct, undefined, 4); + if (flFixBreakage) { + s = s.replace (/\u2028/g,'\\u2028').replace (/\u2029/g,'\\u2029'); + } + return (s); + } function stringAddCommas (x) { //5/27/14 by DW return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); } @@ -481,9 +497,6 @@ function getFavicon (url) { //7/18/14 by DW var domain = getDomain (url); return ("http://www.google.com/s2/favicons?domain=" + domain); }; -function jsonStringify (jstruct) { //7/19/14 by DW - return (JSON.stringify (jstruct, undefined, 4)); - } function getURLParameter (name) { //7/21/14 by DW return (decodeURI ((RegExp(name + '=' + '(.+?)(&|$)').exec(location.search)||[,null])[1])); } diff --git a/package.json b/package.json old mode 100755 new mode 100644 index 456a50f..0d06a58 --- a/package.json +++ b/package.json @@ -1,18 +1,19 @@ { - "name": "PagePark", - "description": "A simple Node.js folder-based HTTP server that serves static and dynamic pages for domains.", - "author": "Dave Winer ", - "version": "0.48.0", - "scripts": { - "start": "node pagepark.js" - }, - "dependencies" : { - "request": "*", - "mime": "*", - "marked": "*" - }, - "license": "MIT", - "engines": { - "node": "0.10.*" - } - } + "name": "PagePark", + "description": "A simple Node.js folder-based HTTP server that serves static and dynamic pages for domains.", + "author": "Dave Winer ", + "version": "0.48.0", + "scripts": { + "start": "node pagepark.js" + }, + "dependencies" : { + "request": "*", + "mime": "*", + "opmlparser": "*", + "marked": "*" + }, + "license": "MIT", + "engines": { + "node": "0.10.*" + } + } diff --git a/pagepark.js b/pagepark.js index b70d816..dbf0a53 100644 --- a/pagepark.js +++ b/pagepark.js @@ -1,5 +1,5 @@ -var myVersion = "0.60c", myProductName = "PagePark"; - +var myVersion = "0.61s", myProductName = "PagePark"; + //The MIT License (MIT) //Copyright (c) 2014 Dave Winer @@ -32,6 +32,7 @@ var marked = require ("marked"); var dns = require ("dns"); var mime = require ("mime"); //1/8/15 by DW var utils = require ("./lib/utils.js"); //1/18/15 by DW +var opmlLib = require ("./lib/opml.js"); //6/23/15 by DW var folderPathFromEnv = process.env.pageparkFolderPath; //1/3/15 by DW @@ -39,7 +40,8 @@ var pageparkPrefs = { myPort: 1339, //1/8/15 by DW -- was 80, see note in readme.md indexFilename: "index", flProcessScriptFiles: true, extScriptFiles: "js", //5/5/15 by DW - flProcessMarkdownFiles: true, extMarkdownFiles: "md" //5/5/15 by DW + flProcessMarkdownFiles: true, extMarkdownFiles: "md", //5/5/15 by DW + flProcessOpmlFiles: true, extOpmlFiles: "opml" //6/23/15 by DW }; var fnamePrefs = "prefs/prefs.json"; @@ -56,7 +58,10 @@ var domainsPath = "domains/"; var configFname = "/config.json"; var mdTemplatePath = "prefs/mdTemplate.txt"; -var urlDefaultTemplate = "http://fargo.io/code/pagepark/defaultmarkdowntemplate.txt"; +var urlDefaultMarkdownTemplate = "http://fargo.io/code/pagepark/defaultmarkdowntemplate.txt"; + +var opmlTemplatePath = "prefs/opmlTemplate.txt"; +var urlDefaultOpmlTemplate = "http://fargo.io/code/pagepark/defaultopmltemplate.txt"; function fsSureFilePath (path, callback) { var splits = path.split ("/"); @@ -114,12 +119,12 @@ function getFullFilePath (relpath) { //1/3/15 by DW } return (folderpath + relpath); } -function getMarkdownTemplate (callback) { - var f = getFullFilePath (mdTemplatePath); +function getTemplate (myTemplatePath, urlDefaultTemplate, callback) { + var f = getFullFilePath (myTemplatePath); fs.readFile (f, function (err, data) { if (err) { httpReadUrl (urlDefaultTemplate, function (s) { - fs.writeFile (mdTemplatePath, s, function (err) { + fs.writeFile (myTemplatePath, s, function (err) { if (callback != undefined) { callback (s); } @@ -133,6 +138,12 @@ function getMarkdownTemplate (callback) { } }); } +function getMarkdownTemplate (callback) { + getTemplate (mdTemplatePath, urlDefaultMarkdownTemplate, callback); + } +function getOpmlTemplate (callback) { //6/23/15 by DW + getTemplate (opmlTemplatePath, urlDefaultOpmlTemplate, callback); + } function checkPathForIllegalChars (path) { function isIllegal (ch) { if (utils.isAlpha (ch) || utils.isNumeric (ch)) { @@ -161,6 +172,20 @@ function everySecond () { } } function handleHttpRequest (httpRequest, httpResponse) { + function hasAcceptHeader (theHeader) { + if (httpRequest.headers.accept === undefined) { + return (false); + } + else { + var split = httpRequest.headers.accept.split (", "); + for (var i = 0; i < split.length; i++) { + if (split [i] == theHeader) { + return (true); + } + } + return (false); + } + } function getDomainFolder (host, callback) { //5/11/15 by DW var folder = getFullFilePath (domainsPath); var domainfolder = folder + host; @@ -187,8 +212,10 @@ function handleHttpRequest (httpRequest, httpResponse) { urlSiteContents: undefined, flProcessScriptFiles: true, flProcessMarkdownFiles: true, + flProcessOpmlFiles: true, extScriptFiles: pageparkPrefs.extScriptFiles, - extMarkdownFiles: pageparkPrefs.extMarkdownFiles + extMarkdownFiles: pageparkPrefs.extMarkdownFiles, + extOpmlFiles: pageparkPrefs.extOpmlFiles }; var f = getFullFilePath (domainsPath) + host + configFname; fs.readFile (f, function (err, data) { @@ -234,12 +261,10 @@ function handleHttpRequest (httpRequest, httpResponse) { httpResponse.writeHead (200, {"Content-Type": type}); httpResponse.end (val.toString ()); } - function defaultReturn (type, data) { httpResponse.writeHead (200, {"Content-Type": type}); httpResponse.end (data); } - fs.readFile (f, function (err, data) { if (err) { return404 (); @@ -280,6 +305,24 @@ function handleHttpRequest (httpRequest, httpResponse) { defaultReturn (type, data); } break; + case config.extOpmlFiles: //6/23/15 by DW + var flReturnHtml = !hasAcceptHeader ("text/x-opml"); + if (pageparkPrefs.flProcessOpmlFiles && config.flProcessOpmlFiles && flReturnHtml) { //xxx + getOpmlTemplate (function (theTemplate) { + var opmltext = data.toString (), pagetable = new Object (); + opmlLib.readOpmlString (opmltext, function (theOutline) { + pagetable.bodytext = utils.jsonStringify (theOutline); + pagetable.title = utils.stringLastField (f, "/"); + var s = utils.multipleReplaceAll (theTemplate, pagetable, false, "[%", "%]"); + httpResponse.writeHead (200, {"Content-Type": "text/html"}); + httpResponse.end (s); + }); + }); + } + else { + defaultReturn (type, data); + } + break; default: defaultReturn (type, data); break;