dx: test/finish bot preview

.circleci/config.yml

@@ -12,7 +12,7 @@ jobs:
       - run: "./scripts/pr-parser-preview.sh"
 
       - store_artifacts:
-          path: /tmp/artifacts
+          path: tmp/artifacts
 
   test-web:
     docker:

package.json

@@ -26,6 +26,7 @@
   "author": "Postlight",
   "license": "ISC",
   "devDependencies": {
+    "@jesses/circle-github-bot": "^2.1.0",
     "all-contributors-cli": "^3.0.7",
     "babel-eslint": "^7.1.0",
     "babel-jest": "^16.0.0",
@@ -45,7 +46,6 @@
     "brfs-babel": "^1.0.0",
     "browserify": "^13.1.1",
     "changelog-maker": "^2.3.0",
-    "circle-github-bot": "^2.0.1",
     "eslint": "^3.8.1",
     "eslint-config-airbnb": "^12.0.0",
     "eslint-import-resolver-babel-module": "^2.0.1",

scripts/comment-on-pr.js

@@ -1,28 +1,65 @@
 /* eslint-disable */
-const bot = require('circle-github-bot').create();
+const bot = require('@jesses/circle-github-bot').default.create();
 const Mercury = require('../dist/mercury.js');
 const fs = require("fs");
 
-const run = async () => {
-  const screenshot = process.argv[2];
+const run = () => {
+  const screenshotPath = process.argv[2];
 
-  const fixture = screenshot.split('tmp/artifacts/')[1].slice(0, -4);
+  const fixture = screenshotPath.split('tmp/artifacts/')[1].slice(0, -4);
 
   const html = fs.readFileSync(`${fixture}`);
-  const json = await Mercury.parse('http://example.com', html, { fallback: false });
 
-  const jsonPath = `${screenshot}-parsed.json`;
-  const fixtureArtifactPath = "tmp/artifacts/" + fixture;
+  // first parse is just to get the url
+  Mercury.parse('http://example.com', html, { fallback: false }).then(({ url, domain, excerpt, word_count, direction }) => {
+    // with the url, second pass will test the correct parser
+    Mercury.parse(url, html, { fallback: false }).then(json => {
+      // removing excerpt b/c this comes from content, not necessary
+      delete json.excerpt
 
-  fs.writeFileSync(jsonPath, JSON.stringify(json));
-  fs.writeFileSync(fixtureArtifactPath, html);
+      // adding items that aren't pulled in custom parser w/out fallback
+      Object.assign(json, { url, domain, word_count, direction });
 
-  bot.comment(`
-  <h3>${bot.env.commitMessage}</h3>
-  Page: <strong>${bot.artifactLink(ffixtureArtifactPath, 'Page')}</strong>
-  Screenshot: <strong>${bot.artifactLink(screenshot, 'Screenshot')}</strong>
-  JSON: <strong>${bot.artifactLink(jsonPath, 'JSON')}</strong>
-  `);
+      // a quick preview of the parsed content in an html file
+      const previewHtml = `<h1>${json.title}</h1><p>${json.author}</p>${json.content}`
+
+
+      const jsonPath = `${screenshotPath}-parsed.json`;
+      const fixtureArtifactPath = `tmp/artifacts/${fixture}`;
+      const previewPath = `tmp/artifacts/${fixture}.preview.html`;
+
+      fs.writeFileSync(previewPath, previewHtml);
+      fs.writeFileSync(jsonPath, JSON.stringify(json));
+      fs.writeFileSync(fixtureArtifactPath, html);
+
+      bot.comment(process.env.GH_AUTH_TOKEN, `### 🤖 Automated Parsing Preview 🤖
+**Commit:** \`${bot.env.commitMessage}\`
+
+![Screenshot of fixture (this embed should work after repo is public)](${bot.artifactUrl(screenshotPath)})
+
+**${bot.artifactLink(fixtureArtifactPath, 'HTML Fixture')}**
+
+**${bot.artifactLink(previewPath, 'Parsed Content Preview')}**
+
+<details>
+<summary><b>Parsed JSON</b></summary>
+
+\`\`\`json
+${JSON.stringify(json, null, 2)}
+\`\`\`
+
+</details>
+<br />
+
+
+**\`null\` fields**
+
+${Object.keys(json).map(key => json[key] !== null ? '' : `  * \`${key}\n\``).join('') || 'None'}
+
+`
+      );
+    });
+  });
 }
 
 run();

scripts/generate-fixture-preview.js

@@ -33,6 +33,7 @@ function capturePage() {
     page.evaluateJavaScript(script);
 
     var filename = new Date();
+    page.clipRect = { top: 0, left: 0, width: 1366, height: 768 };
     page.render('tmp/artifacts/' + fixturePath + '.png');
     pageRenderComplete();
   });

scripts/pr-parser-preview.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-changes=( `git diff master --name-only` )
+changes=( `git diff origin/master --name-only` )
 
 for fixture in "${changes[@]}"
 do

yarn.lock

@@ -2,6 +2,10 @@
 # yarn lockfile v1
 
 
+"@jesses/circle-github-bot@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@jesses/circle-github-bot/-/circle-github-bot-2.1.0.tgz#0f43429ff7bc17a962812c22adb25b890367b16d"
+
 JSONStream@^1.0.3:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.2.1.tgz#32aa5790e799481083b49b4b7fa94e23bae69bf9"
@@ -1270,10 +1274,6 @@ cipher-base@^1.0.0, cipher-base@^1.0.1:
   dependencies:
     inherits "^2.0.1"
 
-circle-github-bot@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/circle-github-bot/-/circle-github-bot-2.0.1.tgz#feefa4c7f788ce842307e00d290d264728689f69"
-
 circular-json@^0.3.0:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/circular-json/-/circular-json-0.3.1.tgz#be8b36aefccde8b3ca7aa2d6afc07a37242c0d2d"