initial

.gitignore

@@ -0,0 +1,16 @@
+# ---> Go
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+*.log
+
+
+# Binary
+hugobot
+
+# Sqlite
+*.sqlite-*

Dockerfile

@@ -0,0 +1,52 @@
+FROM golang:1.11-alpine as builder
+
+MAINTAINER Chakib <contact@bitcointechweekly.com>
+
+# Copy source
+COPY . /go/src/hugobot
+
+# install dependencies and build
+RUN apk add --no-cache --upgrade \
+    ca-certificates \
+    git \
+    openssh \
+    make \
+    alpine-sdk
+
+RUN cd /go/src/hugobot \
+&&  make install
+
+################################
+#### FINAL IMAGE
+###############################
+
+
+FROM alpine as final
+
+ENV WEBSITE_PATH=/website
+ENV HUGOBOT_DB_PATH=/db
+
+RUN apk add --no-cache --upgrade \
+    ca-certificates \
+    bash \
+    sqlite \
+    jq
+
+COPY --from=builder /go/bin/hugobot /bin/
+
+
+RUN mkdir -p ${HUGOBOT_DB_PATH}
+RUN mkdir -p ${WEBSITE_PATH}
+
+
+VOLUME ${HUGOBOT_DB_PATH}
+
+
+# Expose API ports
+EXPOSE 8734
+
+# copy entrypoint
+COPY "docker-entrypoint.sh" /entry
+
+ENTRYPOINT ["/entry"]
+CMD ["hugobot", "server"]

Dockerfile-sqliteweb

@@ -0,0 +1,9 @@
+FROM coleifer/sqlite
+RUN apk add --no-cache --virtual .build-reqs build-base gcc make \
+    && pip install --no-cache-dir cython \
+    && pip install --no-cache-dir flask peewee sqlite-web \
+    && apk del .build-reqs
+EXPOSE 8080
+VOLUME /db
+WORKDIR /db
+CMD sqlite_web -H 0.0.0.0 -x $SQLITE_DATABASE -P

Makefile

@@ -0,0 +1,22 @@
+TARGET=hugobot
+
+GOINSTALL := GO111MODULE=on go install -v
+GOBUILD := GO111MODULE=on go build -v
+PKG := hugobot
+
+.PHONY: all build install
+
+
+all: build
+
+build:
+	$(GOBUILD) -o $(TARGET)
+
+install:
+	$(GOINSTALL)
+
+
+
+
+
+

README.md

@@ -0,0 +1,47 @@
+**MIRRORED FROM**: https://git.sp4ke.com/sp4ke/hugobot
+
+# HUGOBOT
+
+*hugobot* is a an automated content fetch and aggregation bot for [Hugo][hugo] data
+driven websites. It has the following features:
+
+
+## Data fetch
+
+- Add feeds to the bot in the `feeds` sqlite table
+- Currently handles these types of feeds: `RSS`, `Github Releases`, `Newsletters`
+- Define your own feed types by implementing the `JobHandler` interface (see
+  `handlers/handlers.go`).
+- Hugobot automatically fetch new posts from the feeds you defined
+- It runs periodically to download new posts in the defined feeds.
+- Everything is saved on an sqlite
+- The scheduler can handle any number of tasks and uses leveldb for
+  caching/resuming jobs.
+
+
+## Hugo export
+
+- Data is automatically exported to the configured Hugo website path.
+- It can export `markdwon` files or `json/toml` data files
+- All fields in the exported files can be customized
+- You can define custom output formats by using the `FormatHandler` interface.
+
+
+## API
+
+- *hugobot* also includes a webserver API that can be used with Hugo [Data
+  Driven Mode][data-driven].
+
+- WIP: Insert and query data 
+
+- An example usage is the automated generation of Bitcoin addresses for new
+  articles on [bitcointechweekly.com][btw-btc]
+
+## Sqliteweb interface
+
+- See Docker files
+
+
+[data-driven]:https://gohugo.io/templates/data-templates/#data-driven-content
+[btw-btc]:https://bitcointechweekly.com/btc/3Jv15g4G5LDnBJPDh1e2ja8NPnADzMxhVh
+[hugo]:https://gohugo.io

api.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"io"
+	"os"
+	"strconv"
+
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+
+	"git.sp4ke.com/sp4ke/hugobot/v3/config"
+
+	"git.sp4ke.com/sp4ke/hugobot/v3/bitcoin"
+
+	gum "git.sp4ke.com/sp4ke/gum.git"
+	"github.com/gin-gonic/gin"
+)
+
+var (
+	apiLogFile *os.File
+)
+
+type API struct {
+	router *gin.Engine
+}
+
+func (api *API) Run(m gum.UnitManager) {
+
+	feedsRoute := api.router.Group("/feeds")
+	{
+		feedCtrl := &feeds.FeedCtrl{}
+
+		feedsRoute.POST("/", feedCtrl.Create)
+		feedsRoute.DELETE("/:id", feedCtrl.Delete)
+		feedsRoute.GET("/", feedCtrl.List) // Get all
+		//feedsRoute.Get("/:id", feedCtrl.GetById) // Get one
+	}
+
+	btcRoute := api.router.Group("/btc")
+	{
+		btcRoute.GET("/address", bitcoin.GetAddressCtrl)
+	}
+
+	// Run router
+	go func() {
+
+		err := api.router.Run(":" + strconv.Itoa(config.C.ApiPort))
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	// Wait for stop signal
+	<-m.ShouldStop()
+
+	// Shutdown
+	api.Shutdown()
+	m.Done()
+}
+
+func (api *API) Shutdown() {}
+
+func NewApi() *API {
+	apiLogFile, _ = os.Create(".api.log")
+	gin.DefaultWriter = io.MultiWriter(apiLogFile, os.Stdout)
+
+	api := &API{
+		router: gin.Default(),
+	}
+
+	return api
+}

bitcoin/addresses.go

@@ -0,0 +1,188 @@
+package bitcoin
+
+import (
+	"database/sql"
+	"log"
+	"net/http"
+
+	"git.sp4ke.com/sp4ke/hugobot/v3/db"
+
+	"github.com/gin-gonic/gin"
+	sqlite3 "github.com/mattn/go-sqlite3"
+)
+
+var DB = db.DB
+
+const (
+	DBBTCAddressesSchema = `CREATE TABLE IF NOT EXISTS btc_addresses (
+		addr_id INTEGER PRIMARY KEY,
+		address TEXT NOT NULL UNIQUE,
+		address_position INTEGER NOT NULL DEFAULT 0,
+		linked_article_title TEXT DEFAULT '',
+		linked_article_id TEXT NOT NULL DEFAULT '',
+		used INTEGER NOT NULL DEFAULT 0,
+		synced INTEGER NOT NULL DEFAULT 0
+	)`
+
+	QueryUnusedAddress = `SELECT * FROM btc_addresses WHERE used = 0 LIMIT 1 `
+
+	UpdateAddressQuery = `UPDATE btc_addresses 
+		SET linked_article_id = ?,
+		linked_article_title = ?,
+		used = ?
+		WHERE addr_id = ?
+	`
+)
+
+type BTCAddress struct {
+	ID                 int64  `db:"addr_id"`
+	Address            string `db:"address"`
+	AddrPosition       int64  `db:"address_position"`
+	LinkedArticleTitle string `db:"linked_article_title"`
+	LinkedArticleID    string `db:"linked_article_id"`
+	Used               bool   `db:"used"`
+	Synced             bool   `db:"synced"`
+}
+
+// TODO: Set address to synced
+func (a *BTCAddress) SetSynced() error {
+	a.Synced = true
+	query := `UPDATE btc_addresses SET synced = :synced WHERE addr_id = :addr_id`
+	_, err := DB.Handle.NamedExec(query, a)
+	if err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func GetAddressByPos(pos int) (*BTCAddress, error) {
+	var btcAddr BTCAddress
+	err := DB.Handle.Get(&btcAddr,
+		"SELECT * FROM btc_addresses WHERE address_position = ?",
+		pos,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &btcAddr, nil
+}
+
+func GetAddressByArticleID(artId string) (*BTCAddress, error) {
+	var btcAddr BTCAddress
+	err := DB.Handle.Get(&btcAddr,
+		"SELECT * FROM btc_addresses WHERE linked_article_id = ?",
+		artId,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &btcAddr, nil
+}
+
+func GetAllUsedUnsyncedAddresses() ([]*BTCAddress, error) {
+	var addrs []*BTCAddress
+	err := DB.Handle.Select(&addrs,
+		"SELECT * FROM btc_addresses WHERE used = 1 AND synced = 0",
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return addrs, nil
+}
+
+func GetNextUnused() (*BTCAddress, error) {
+	var btcAddr BTCAddress
+	err := DB.Handle.Get(&btcAddr, QueryUnusedAddress)
+	if err != nil {
+		return nil, err
+	}
+	return &btcAddr, nil
+}
+
+func GetAddressForArticle(artId string, artTitle string) (*BTCAddress, error) {
+	// Check if article already has an assigned address
+	addr, err := GetAddressByArticleID(artId)
+	sqliteErr, isSqliteErr := err.(sqlite3.Error)
+
+	if (isSqliteErr && sqliteErr.Code != sqlite3.ErrNotFound) ||
+		(err != nil && !isSqliteErr && err != sql.ErrNoRows) {
+
+		log.Println("err")
+		return nil, err
+	}
+
+	if err == nil {
+		// If different title update it
+		if artTitle != addr.LinkedArticleTitle {
+			addr.LinkedArticleTitle = artTitle
+			// Store newly assigned address
+			_, err = DB.Handle.Exec(UpdateAddressQuery,
+				addr.LinkedArticleID,
+				addr.LinkedArticleTitle,
+				addr.Used,
+				addr.ID,
+			)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return addr, nil
+	}
+
+	// Get next unused address
+	addr, err = GetNextUnused()
+	if err != nil {
+		return nil, err
+	}
+
+	addr.LinkedArticleID = artId
+	addr.LinkedArticleTitle = artTitle
+	addr.Used = true
+
+	// Store newly assigned address
+	_, err = DB.Handle.Exec(UpdateAddressQuery,
+		addr.LinkedArticleID,
+		addr.LinkedArticleTitle,
+		addr.Used,
+		addr.ID,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return addr, nil
+}
+
+func GetAddressCtrl(c *gin.Context) {
+	artId := c.Query("articleId")
+	artTitle := c.Query("articleTitle")
+
+	addr, err := GetAddressForArticle(artId, artTitle)
+
+	if err != nil {
+		c.JSON(http.StatusBadRequest,
+			gin.H{"status": http.StatusBadRequest,
+				"error": err.Error()})
+		c.Abort()
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"status": http.StatusOK,
+		"addr":   addr.Address,
+	})
+
+}
+
+func init() {
+	_, err := DB.Handle.Exec(DBBTCAddressesSchema)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

commands.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/export"
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/static"
+	"log"
+
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var startServerCmd = cli.Command{
+	Name:    "server",
+	Aliases: []string{"s"},
+	Usage:   "Run server",
+	Action:  startServer,
+}
+
+var exportCmdGrp = cli.Command{
+	Name:    "export",
+	Aliases: []string{"e"},
+	Usage:   "Export to hugo",
+	Subcommands: []cli.Command{
+		exportPostsCmd,
+		exportWeeksCmd,
+		exportBTCAddressesCmd,
+	},
+}
+
+var exportBTCAddressesCmd = cli.Command{
+	Name:   "btc",
+	Usage:  "export bitcoin addresses",
+	Action: exportAddresses,
+}
+
+var exportWeeksCmd = cli.Command{
+	Name:   "weeks",
+	Usage:  "export weeks",
+	Action: exportWeeks,
+}
+
+var exportPostsCmd = cli.Command{
+	Name:   "posts",
+	Usage:  "Export posts to hugo",
+	Action: exportPosts,
+}
+
+func startServer(c *cli.Context) {
+	server()
+}
+
+func exportPosts(c *cli.Context) {
+	exporter := export.NewHugoExporter()
+	feeds, err := feeds.ListFeeds()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, f := range feeds {
+		exporter.Export(*f)
+	}
+
+	// Export static data
+	err = static.HugoExportData()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}
+
+func exportWeeks(c *cli.Context) {
+	err := export.ExportWeeks()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}
+
+func exportAddresses(c *cli.Context) {
+	err := export.ExportBTCAddresses()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}

config.toml

@@ -0,0 +1 @@
+api-port = 8734

config/config.go

@@ -0,0 +1,58 @@
+package config
+
+import (
+	"log"
+	"path"
+
+	"github.com/fatih/structs"
+)
+
+const (
+	BTCQRCodesDir = "qrcodes"
+)
+
+type Config struct {
+	WebsitePath               string
+	GithubAccessToken         string
+	RelBitcoinAddrContentPath string
+	ApiPort                   int
+}
+
+var (
+	C *Config
+)
+
+func HugoData() string {
+	return path.Join(C.WebsitePath, "data")
+}
+
+func HugoContent() string {
+	return path.Join(C.WebsitePath, "content")
+}
+
+func RelBitcoinAddrContentPath() string {
+	return path.Join(C.WebsitePath, C.RelBitcoinAddrContentPath)
+}
+
+func RegisterConf(conf string, val interface{}) error {
+	log.Printf("Setting %#v to %#v", conf, val)
+	s := structs.New(C)
+
+	field, ok := s.FieldOk(conf)
+
+	// Conf option not registered in Config struct
+	if !ok {
+		return nil
+	}
+
+	err := field.Set(val)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func init() {
+	C = new(Config)
+}

db/db.go

@@ -0,0 +1,72 @@
+package db
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+const (
+	DBName        = "hugobot.sqlite"
+	DBPragma      = ` PRAGMA foreign_keys = ON; `
+	DBBasePathEnv = "HUGOBOT_DB_PATH"
+)
+
+var (
+	DBOptions = map[string]string{
+		"_journal_mode": "WAL",
+	}
+
+	DB *Database
+)
+
+type Database struct {
+	Handle *sqlx.DB
+}
+
+func (d *Database) Open() error {
+
+	dsnOptions := &url.Values{}
+	for k, v := range DBOptions {
+		dsnOptions.Set(k, v)
+	}
+
+	// Get db base path
+	path, set := os.LookupEnv(DBBasePathEnv)
+	if !set {
+		path = "."
+	}
+	path = filepath.Join(path, DBName)
+	//path = fmt.Sprintf("%s/%s", path, DBName)
+
+	dsn := fmt.Sprintf("file:%s?%s", path, dsnOptions.Encode())
+
+	log.Printf("Opening sqlite db %s\n", dsn)
+
+	var err error
+	d.Handle, err = sqlx.Open("sqlite3", dsn)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Execute Pragmas
+	d.Handle.MustExec(DBPragma)
+
+	return nil
+}
+
+type AutoIncr struct {
+	ID      int64     `json:"id"`
+	Created time.Time `json:"created"`
+}
+
+func init() {
+	DB = &Database{}
+	DB.Open()
+}

docker-compose.yml

@@ -0,0 +1,42 @@
+version: "2.2"
+
+volumes:
+    js-deps:
+    build:
+    sqlite-db:
+
+services:
+    bot:
+        image: hugobot/hugobot
+        build: .
+
+        volumes:
+            - path_to_website:/website
+            - $PWD:/hugobot
+            - sqlite-db:/db
+
+        environment:
+            - BUILD_DIR=/build
+
+        restart: on-failure
+
+        ports:
+            - "8734:8734"
+
+        working_dir: /hugobot
+
+
+    sqlite-web:
+        image: hugobot/sqlite-web
+        build:
+            context: .
+            dockerfile: ./Dockerfile-sqliteweb
+        ports:
+            - "8080"
+        volumes:
+            - sqlite-db:/db
+
+        environment:
+            - SQLITE_DATABASE=hugobot.sqlite
+            - SQLITE_WEB_PASSWORD=hugobot
+

docker-entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "$(ls -A "$HUGOBOT_DB_PATH")" ]];then
+    echo "WARNING !! $HUGOBOT_DB_PATH is empty, creating new database !"
+fi
+
+if [[ -z "$(ls -A "$WEBSITE_PATH")" ]];then
+    echo "you need to mount the website path !"
+    exit 1
+fi
+
+
+exec "$@"

encoder/encoder.go

@@ -0,0 +1,58 @@
+package encoder
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	JSON = iota
+	TOML
+)
+
+type Encoder interface {
+	Encode(v interface{}) error
+}
+
+type ExportEncoder struct {
+	encoder Encoder
+	w       io.Writer
+	eType   int
+}
+
+func (ee *ExportEncoder) Encode(v interface{}) error {
+	var err error
+
+	if ee.eType == TOML {
+		fmt.Fprintf(ee.w, "+++\n")
+	}
+
+	err = ee.encoder.Encode(v)
+
+	if ee.eType == TOML {
+		fmt.Fprintf(ee.w, "+++\n")
+	}
+
+	return err
+}
+
+func NewExportEncoder(w io.Writer, encType int) *ExportEncoder {
+
+	var enc Encoder
+
+	switch encType {
+	case JSON:
+		enc = json.NewEncoder(w)
+	case TOML:
+		enc = toml.NewEncoder(w)
+	}
+
+	return &ExportEncoder{
+		encoder: enc,
+		w:       w,
+		eType:   encType,
+	}
+}

export/btc_addresses.go

@@ -0,0 +1,65 @@
+package export
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/bitcoin"
+	"git.sp4ke.com/sp4ke/hugobot/v3/config"
+	"git.sp4ke.com/sp4ke/hugobot/v3/encoder"
+	"log"
+	"os"
+	"path/filepath"
+
+	qrcode "github.com/skip2/go-qrcode"
+)
+
+func ExportBTCAddresses() error {
+	unusedAddrs, err := bitcoin.GetAllUsedUnsyncedAddresses()
+	if err != nil {
+		return err
+	}
+
+	for _, a := range unusedAddrs {
+		//first export the qr codes
+		log.Println("exporting ", a)
+
+		qrFileName := a.Address + ".png"
+
+		qrCodePath := filepath.Join(config.RelBitcoinAddrContentPath(),
+			config.BTCQRCodesDir, qrFileName)
+
+		err := qrcode.WriteFile(a.Address, qrcode.Medium, 580, qrCodePath)
+		if err != nil {
+			return err
+		}
+
+		// store the address pages
+
+		filename := a.Address + ".md"
+		filePath := filepath.Join(config.RelBitcoinAddrContentPath(), filename)
+
+		data := map[string]interface{}{
+			"linked_article_id": a.LinkedArticleID,
+			//"resources": []map[string]interface{}{
+			//map[string]interface{}{
+			//"src": filepath.Join(config.BTCQRCodesDir, a.Address+".png"),
+			//},
+			//},
+		}
+
+		addressPage, err := os.Create(filePath)
+		if err != nil {
+			return err
+		}
+
+		tomlExporter := encoder.NewExportEncoder(addressPage, encoder.TOML)
+		tomlExporter.Encode(data)
+
+		// Set synced
+		err = a.SetSynced()
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}

export/bulletin.go

@@ -0,0 +1,17 @@
+package export
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+	"strings"
+)
+
+func BulletinExport(exp Map, feed feeds.Feed, post posts.Post) error {
+
+	bulletinInfo := strings.Split(feed.Section, "/")
+
+	if bulletinInfo[0] == "bulletin" {
+		exp["bulletin_type"] = bulletinInfo[1]
+	}
+	return nil
+}

export/export.go

@@ -0,0 +1,214 @@
+package export
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"git.sp4ke.com/sp4ke/hugobot/v3/config"
+	"git.sp4ke.com/sp4ke/hugobot/v3/encoder"
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/filters"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+	"git.sp4ke.com/sp4ke/hugobot/v3/types"
+	"git.sp4ke.com/sp4ke/hugobot/v3/utils"
+)
+
+var PostMappers []PostMapper
+var FeedMappers []FeedMapper
+
+type Map map[string]interface{}
+
+type PostMapper func(Map, feeds.Feed, posts.Post) error
+type FeedMapper func(Map, feeds.Feed) error
+
+// Exported version of a post
+type PostExport struct {
+	ID        int64     `json:"id"`
+	Title     string    `json:"title"`
+	Link      string    `json:"link"`
+	Published time.Time `json:"published"`
+	Content   string    `json:"content"`
+}
+
+type PostMap map[int64]Map
+
+type FeedExport struct {
+	Name       string           `json:"name"`
+	Section    string           `json:"section"`
+	Categories types.StringList `json:"categories"`
+	Posts      PostMap          `json:"posts"`
+}
+
+type HugoExporter struct{}
+
+func (he HugoExporter) Handle(feed feeds.Feed) error {
+	return he.export(feed)
+}
+
+func (he HugoExporter) export(feed feeds.Feed) error {
+	log.Printf("Exporting %s to %s", feed.Name, config.HugoData())
+
+	posts, err := posts.GetPostsByFeedId(feed.FeedID)
+	if err != nil {
+		return err
+	}
+
+	if len(posts) == 0 {
+		log.Printf("nothing to export")
+		return nil
+	}
+
+	// Run filters on posts
+	for _, p := range posts {
+		filters.RunPostFilterHooks(feed, p)
+	}
+
+	// Dir and filename
+	dirPath := filepath.Join(config.HugoData(), feed.Section)
+	cleanFeedName := strings.Replace(feed.Name, "/", "-", -1)
+	filePath := filepath.Join(dirPath, cleanFeedName+".json")
+
+	err = utils.Mkdir(dirPath)
+	if err != nil {
+		return err
+	}
+
+	feedExp := Map{
+		"name":       feed.Name,
+		"section":    feed.Section,
+		"categories": feed.Categories,
+	}
+
+	runFeedMappers(feedExp, feed)
+
+	postsMap := make(PostMap)
+	for _, p := range posts {
+		exp := Map{
+			"id":        p.PostID,
+			"title":     p.Title,
+			"link":      p.Link,
+			"published": p.Published,
+			"updated":   p.Updated,
+			//"content":   p.Content,
+		}
+		runPostMappers(exp, feed, *p)
+
+		postsMap[p.PostID] = exp
+	}
+	feedExp["posts"] = postsMap
+
+	outputFile, err := os.Create(filePath)
+	defer outputFile.Close()
+	if err != nil {
+		return err
+	}
+
+	exportEncoder := encoder.NewExportEncoder(outputFile, encoder.JSON)
+	exportEncoder.Encode(feedExp)
+	//jsonEnc.Encode(feedExp)
+
+	// Handle feeds which export posts individually as hugo posts
+	// Like bulletin
+
+	if feed.ExportPosts {
+		for _, p := range posts {
+
+			exp := map[string]interface{}{
+				"id":           p.PostID,
+				"title":        p.Title,
+				"name":         feed.Name,
+				"author":       p.Author,
+				"description":  p.PostDescription,
+				"externalLink": feed.UseExternalLink,
+				"display_name": feed.DisplayName,
+				"publishdate":  p.Published,
+				"date":         p.Updated,
+				"issuedate":    utils.NextThursday(p.Updated),
+				"use_data":     true,
+				"slug":         p.ShortID,
+				"link":         p.Link,
+				// Content is written in the post
+				"content":    p.Content,
+				"categories": feed.Categories,
+				"tags":       strings.Split(p.Tags, ","),
+			}
+
+			if feed.Publications != "" {
+				exp["publications"] = strings.Split(feed.Publications, ",")
+			}
+
+			runPostMappers(exp, feed, *p)
+
+			dirPath := filepath.Join(config.HugoContent(), feed.Section)
+			cleanFeedName := strings.Replace(feed.Name, "/", "-", -1)
+			fileName := fmt.Sprintf("%s-%s.md", cleanFeedName, p.ShortID)
+			filePath := filepath.Join(dirPath, fileName)
+
+			outputFile, err := os.Create(filePath)
+
+			defer outputFile.Close()
+
+			if err != nil {
+				return err
+			}
+
+			exportEncoder := encoder.NewExportEncoder(outputFile, encoder.TOML)
+			exportEncoder.Encode(exp)
+		}
+	}
+	return nil
+}
+
+// Runs in goroutine
+func (he HugoExporter) Export(feed feeds.Feed) {
+	err := he.export(feed)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func NewHugoExporter() HugoExporter {
+	// Make sure path exists
+	err := utils.Mkdir(config.HugoData())
+	if err != nil {
+		log.Fatal(err)
+	}
+	return HugoExporter{}
+}
+
+func runPostMappers(e Map, f feeds.Feed, p posts.Post) {
+	for _, fn := range PostMappers {
+		err := fn(e, f, p)
+		if err != nil {
+			log.Print(err)
+		}
+	}
+}
+
+func runFeedMappers(e Map, f feeds.Feed) {
+	for _, fn := range FeedMappers {
+		err := fn(e, f)
+		if err != nil {
+			log.Print(err)
+		}
+	}
+}
+
+func RegisterPostMapper(mapper PostMapper) {
+	PostMappers = append(PostMappers, mapper)
+}
+
+func RegisterFeedMapper(mapper FeedMapper) {
+	FeedMappers = append(FeedMappers, mapper)
+}
+
+func init() {
+	RegisterPostMapper(BulletinExport)
+	RegisterPostMapper(NewsletterPostLayout)
+	RegisterPostMapper(RFCExport)
+	RegisterPostMapper(ReleaseExport)
+}

export/newsletters.go

@@ -0,0 +1,18 @@
+package export
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+	"path"
+
+	"github.com/gobuffalo/flect"
+)
+
+func NewsletterPostLayout(exp Map, feed feeds.Feed, post posts.Post) error {
+	section := path.Base(flect.Singularize(feed.Section))
+	if feed.Section == "bulletin/newsletters" {
+		exp["layout"] = section
+	}
+
+	return nil
+}

export/release.go

@@ -0,0 +1,14 @@
+package export
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+)
+
+//
+func ReleaseExport(exp Map, feed feeds.Feed, post posts.Post) error {
+	if feed.Section == "bulletin/releases" {
+		exp["data"] = post.JsonData
+	}
+	return nil
+}

export/rfc.go

@@ -0,0 +1,15 @@
+package export
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+)
+
+// TODO: This happend in the main export file
+func RFCExport(exp Map, feed feeds.Feed, post posts.Post) error {
+	if feed.Section == "bulletin/rfc" {
+		exp["data"] = post.JsonData
+
+	}
+	return nil
+}

export/weeks.go

@@ -0,0 +1,56 @@
+// Export all weeks to the weeks content directory
+package export
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/config"
+	"git.sp4ke.com/sp4ke/hugobot/v3/encoder"
+	"git.sp4ke.com/sp4ke/hugobot/v3/utils"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+const (
+	FirstWeek = "2017-12-07"
+)
+
+var (
+	WeeksContentDir = "weeks"
+)
+
+type WeekData struct {
+	Title string
+	Date  time.Time
+}
+
+func ExportWeeks() error {
+	firstWeek, err := time.Parse("2006-01-02", FirstWeek)
+	if err != nil {
+		return err
+	}
+
+	WeeksTilNow := utils.GetAllThursdays(firstWeek, time.Now())
+	for _, week := range WeeksTilNow {
+		weekName := week.Format("2006-01-02")
+		fileName := weekName + ".md"
+
+		weekFile, err := os.Create(filepath.Join(config.HugoContent(),
+			WeeksContentDir,
+			fileName))
+
+		if err != nil {
+			return err
+		}
+
+		weekData := WeekData{
+			Title: weekName,
+			Date:  week,
+		}
+
+		tomlExporter := encoder.NewExportEncoder(weekFile, encoder.TOML)
+
+		tomlExporter.Encode(weekData)
+	}
+
+	return nil
+}

feed_commands.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/handlers"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+	"fmt"
+	"log"
+	"time"
+
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var fetchCmd = cli.Command{
+	Name:    "fetch",
+	Aliases: []string{"f"},
+	Usage:   "Fetch data from feed",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "since",
+			Usage: "Fetch data since `TIME`, defaults to last refresh time",
+		},
+	},
+	Action: fetchFeeds,
+}
+
+var feedsCmdGroup = cli.Command{
+	Name:  "feeds",
+	Usage: "Feeds related commands. default: list feeds",
+	Flags: []cli.Flag{
+		cli.IntFlag{
+			Name:  "id,i",
+			Value: 0,
+			Usage: "Feeds `id`",
+		},
+	},
+	Subcommands: []cli.Command{
+		fetchCmd,
+	},
+	Action: listFeeds,
+}
+
+func fetchFeeds(c *cli.Context) {
+	var result []*posts.Post
+
+	fList, err := getFeeds(c.Parent())
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, f := range fList {
+		var handler handlers.FormatHandler
+		handler = handlers.GetFormatHandler(*f)
+
+		if c.IsSet("since") {
+			// Parse time
+			t, err := time.Parse(time.UnixDate, c.String("since"))
+			if err != nil {
+				log.Fatal(err)
+			}
+			result, err = handler.FetchSince(f.Url, t)
+
+		} else {
+			result, err = handler.FetchSince(f.Url, f.LastRefresh)
+		}
+
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, post := range result {
+			log.Printf("%s (updated: %s)", post.Title, post.Updated)
+		}
+		log.Println("Total: ", len(result))
+
+	}
+
+}
+
+func listFeeds(c *cli.Context) {
+	fList, err := getFeeds(c)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, f := range fList {
+		fmt.Println(f)
+	}
+}
+
+func getFeeds(c *cli.Context) ([]*feeds.Feed, error) {
+	var fList []*feeds.Feed
+	var err error
+
+	if c.IsSet("id") {
+		feed, err := feeds.GetById(c.Int64("id"))
+		if err != nil {
+			return nil, err
+		}
+
+		fList = append(fList, feed)
+	} else {
+		fList, err = feeds.ListFeeds()
+		if err != nil {
+			return nil, err
+		}
+
+	}
+
+	return fList, nil
+
+}

feeds/ctrl.go

@@ -0,0 +1,117 @@
+package feeds
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/types"
+	"log"
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	sqlite3 "github.com/mattn/go-sqlite3"
+)
+
+const (
+	MsgOK = "OK"
+)
+
+var (
+	ErrNotInt = "expected int"
+)
+
+type FeedCtrl struct{}
+
+func (ctrl FeedCtrl) Create(c *gin.Context) {
+
+	var feedForm FeedForm
+	feedModel := new(Feed)
+
+	if err := c.ShouldBindJSON(&feedForm); err != nil {
+		c.JSON(http.StatusNotAcceptable, gin.H{
+			"status":  http.StatusNotAcceptable,
+			"message": "invalid form",
+			"form":    feedForm})
+		c.Abort()
+		return
+	}
+
+	feedModel.Name = feedForm.Name
+	feedModel.Url = feedForm.Url
+	feedModel.Format = feedForm.Format
+	feedModel.Section = feedForm.Section
+	feedModel.Categories = types.StringList(feedForm.Categories)
+
+	err := feedModel.Write()
+
+	if err != nil {
+		log.Println(err)
+		c.JSON(http.StatusNotAcceptable,
+			gin.H{"status": http.StatusNotAcceptable, "error": err.Error()})
+		c.Abort()
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"status": http.StatusOK, "message": MsgOK})
+}
+
+func (ctrl FeedCtrl) List(c *gin.Context) {
+
+	feeds, err := ListFeeds()
+	if err != nil {
+		c.JSON(http.StatusNotAcceptable, gin.H{
+			"error":  err.Error(),
+			"status": http.StatusNotAcceptable,
+		})
+		c.Abort()
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"status": http.StatusOK, "result": feeds})
+
+}
+
+func (ctrl FeedCtrl) Delete(c *gin.Context) {
+
+	id, err := strconv.Atoi(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusNotAcceptable, gin.H{
+			"error":  ErrNotInt,
+			"status": http.StatusNotAcceptable,
+		})
+		c.Abort()
+		return
+	}
+	err = DeleteById(id)
+
+	sqlErr, isSqlErr := err.(sqlite3.Error)
+	if err != nil {
+
+		if isSqlErr {
+
+			c.JSON(http.StatusInternalServerError,
+				gin.H{
+					"error":  sqlErr.Error(),
+					"status": http.StatusInternalServerError,
+				})
+
+		} else {
+
+			var status int
+
+			switch err {
+			case ErrDoesNotExist:
+				status = http.StatusNotFound
+			default:
+				status = http.StatusInternalServerError
+			}
+
+			c.JSON(status,
+				gin.H{"error": err.Error(), "status": status})
+
+		}
+
+		c.Abort()
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"status": http.StatusOK, "message": MsgOK})
+}

feeds/feed.go

@@ -0,0 +1,262 @@
+package feeds
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/db"
+	"git.sp4ke.com/sp4ke/hugobot/v3/types"
+	"errors"
+	"log"
+	"time"
+
+	sqlite3 "github.com/mattn/go-sqlite3"
+)
+
+//sqlite> SELECT feeds.name, url, feed_formats.name AS format_name from feeds JOIN feed_formats ON feeds.format = feed_formats.id;
+//
+var DB = db.DB
+
+const (
+	DBFeedSchema = `CREATE TABLE IF NOT EXISTS feeds (
+		feed_id INTEGER PRIMARY KEY,
+		name TEXT NOT NULL UNIQUE,
+		display_name TEXT DEFAULT '',
+		publications TEXT DEFAULT '',
+		section TEXT DEFAULT '',
+		categories TEXT DEFAULT '',
+		description TEXT DEFAULT '',
+		url TEXT NOT NULL,
+		export_posts INTEGER DEFAULT 0,
+		last_refresh timestamp DEFAULT -1,
+		created timestamp DEFAULT (strftime('%s')),
+		interval INTEGER DEFAULT 60,
+		format INTEGER NOT NULL DEFAULT 0,
+		serial_run INTEGER DEFAULT 0,
+		use_external_link INTEGER DEFAULT 0,
+		FOREIGN KEY (format) REFERENCES feed_formats(id)
+
+
+	)`
+
+	DBFeedFormatsSchema = `CREATE TABLE IF NOT EXISTS feed_formats (
+		id INTEGER PRIMARY KEY,
+		format_name TEXT NOT NULL UNIQUE
+	)`
+)
+
+const (
+	QDeleteFeedById = `DELETE FROM feeds WHERE feed_id = ?`
+	QGetFeed        = `SELECT * FROM feeds WHERE feed_id = ?`
+	QGetFeedByName  = `SELECT * FROM feeds WHERE name = ?`
+	QGetFeedByURL   = `SELECT * FROM feeds WHERE url = ?`
+	QListFeeds      = `SELECT 
+						feeds.feed_id,
+						feeds.name,
+						feeds.display_name,
+						feeds.publications,
+						feeds.section,
+						feeds.categories,
+						feeds.description,
+						feeds.url,
+						feeds.last_refresh,
+						feeds.created,
+						feeds.format,
+						feeds.serial_run,
+						feeds.use_external_link,
+						feeds.interval,
+						feeds.export_posts,
+						feed_formats.format_name
+						 FROM feeds
+						JOIN feed_formats ON feeds.format = feed_formats.id`
+)
+
+var (
+	ErrDoesNotExist  = errors.New("does not exist")
+	ErrAlreadyExists = errors.New("already exists")
+)
+
+type FeedFormat int
+
+// Feed Formats
+const (
+	FormatRSS FeedFormat = iota
+	FormatHTML
+	FormatJSON
+	FormatTweet
+	FormatRFC
+	FormatGHRelease
+)
+
+var FeedFormats = map[FeedFormat]string{
+	FormatRSS:       "RSS",
+	FormatHTML:      "HTML",
+	FormatJSON:      "JSON",
+	FormatTweet:     "TWEET",
+	FormatRFC:       "RFC",
+	FormatGHRelease: "GithubRelease",
+}
+
+type Feed struct {
+	FeedID       int64            `json:"id" db:"feed_id"`
+	Name         string           `json:"name" db:"name"`
+	Section      string           `json:"section,omitempty"`
+	Categories   types.StringList `json:"categories,omitempty"`
+	Description  string           `json:"description"`
+	Url          string           `json:"url"`
+	Format       FeedFormat       `json:"-"`
+	FormatString string           `json:"format" db:"format_name"`
+	LastRefresh  time.Time        `db:"last_refresh" json:"last_refresh"` // timestamp time.Unix()
+	Created      time.Time        `json:"created"`
+	DisplayName  string           `db:"display_name"`
+	Publications string           `json:"-"`
+
+	// This feed's posts should also be exported individually
+	ExportPosts bool `json:"export_posts" db:"export_posts"`
+
+	// Time in seconds between each polling job on the news feed
+	Interval float64 `json:"refresh_interval"`
+
+	Serial bool `json:"serial" db:"serial_run"` // Jobs for this feed should run in series
+
+	// Items which only contain summaries and redirect to external content
+	// like publications and newsletters
+	UseExternalLink bool `json:"use_external_link" db:"use_external_link"`
+}
+
+func (f *Feed) Write() error {
+
+	query := `INSERT INTO feeds
+		(name, section, categories, url, format)
+VALUES(:name, :section, :categories, :url, :format)`
+
+	_, err := DB.Handle.NamedExec(query, f)
+	sqlErr, isSqlErr := err.(sqlite3.Error)
+	if isSqlErr && sqlErr.Code == sqlite3.ErrConstraint {
+		return ErrAlreadyExists
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (f *Feed) UpdateRefreshTime(time time.Time) error {
+	f.LastRefresh = time
+
+	query := `UPDATE feeds SET last_refresh = ? WHERE feed_id = ?`
+	_, err := DB.Handle.Exec(query, f.LastRefresh, f.FeedID)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GetById(id int64) (*Feed, error) {
+
+	var feed Feed
+	err := DB.Handle.Get(&feed, QGetFeed, id)
+	if err != nil {
+		return nil, err
+	}
+
+	feed.FormatString = FeedFormats[feed.Format]
+
+	return &feed, nil
+}
+
+func GetByName(name string) (*Feed, error) {
+
+	var feed Feed
+	err := DB.Handle.Get(&feed, QGetFeedByName, name)
+	if err != nil {
+		return nil, err
+	}
+
+	feed.FormatString = FeedFormats[feed.Format]
+
+	return &feed, nil
+}
+
+func GetByURL(url string) (*Feed, error) {
+
+	var feed Feed
+	err := DB.Handle.Get(&feed, QGetFeedByURL, url)
+	if err != nil {
+		return nil, err
+	}
+
+	feed.FormatString = FeedFormats[feed.Format]
+
+	return &feed, nil
+
+}
+
+func ListFeeds() ([]*Feed, error) {
+	var feeds []*Feed
+	err := DB.Handle.Select(&feeds, QListFeeds)
+	if err != nil {
+		return nil, err
+	}
+
+	return feeds, nil
+}
+
+func DeleteById(id int) error {
+
+	// If id does not exists return warning
+	var feedToDelete Feed
+	err := DB.Handle.Get(&feedToDelete, QGetFeed, id)
+	if err != nil {
+		return ErrDoesNotExist
+	}
+
+	_, err = DB.Handle.Exec(QDeleteFeedById, id)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Returns true if the feed should be refreshed
+func (feed *Feed) ShouldRefresh() (float64, bool) {
+	lastRefresh := feed.LastRefresh
+	delta := time.Since(lastRefresh).Seconds() // Delta since last refresh
+	//log.Printf("%s delta %f >= interval %f ?", feed.Name, delta, feed.Interval)
+	//
+	//
+	//log.Printf("refresh %s in %.0f seconds", feed.Name, feed.Interval-delta)
+	return delta, delta >= feed.Interval
+}
+
+func init() {
+	_, err := DB.Handle.Exec(DBFeedSchema)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = DB.Handle.Exec(DBFeedFormatsSchema)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Populate feed formats
+	query := `INSERT INTO feed_formats (id, format_name) VALUES (?, ?)`
+	for k, v := range FeedFormats {
+		_, err := DB.Handle.Exec(query, k, v)
+		if err != nil {
+
+			sqlErr, ok := err.(sqlite3.Error)
+			if ok && sqlErr.ExtendedCode == sqlite3.ErrConstraintUnique {
+				log.Panic(err)
+			}
+
+			if !ok {
+				log.Panic(err)
+			}
+		}
+
+	}
+
+}

feeds/form.go

@@ -0,0 +1,9 @@
+package feeds
+
+type FeedForm struct {
+	Name       string     `form:"name" binding:"required"`
+	Url        string     `form:"url" binding:"required"`
+	Format     FeedFormat `form:"format"`
+	Categories []string   `form:"categories"`
+	Section    string     `form:"section"`
+}

filters/filters.go

@@ -0,0 +1,26 @@
+package filters
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+	"log"
+)
+
+type FilterHook func(feed feeds.Feed, post *posts.Post) error
+
+var (
+	PostFilters []FilterHook
+)
+
+func RegisterPostFilterHook(hook FilterHook) {
+	PostFilters = append(PostFilters, hook)
+}
+
+func RunPostFilterHooks(feed feeds.Feed, post *posts.Post) {
+	for _, h := range PostFilters {
+		err := h(feed, post)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}

filters/mailchimp_filter.go

@@ -0,0 +1,65 @@
+package filters
+
+import (
+	"git.sp4ke.com/sp4ke/hugobot/v3/feeds"
+	"git.sp4ke.com/sp4ke/hugobot/v3/posts"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const (
+	PreviewTextSel = ".mcnPreviewText"
+)
+
+var (
+	RemoveSelectors = []string{"style", ".footerContainer", "#awesomewrap", "#templatePreheader", "img", "head"}
+)
+
+func mailChimpFilter(feed feeds.Feed, post *posts.Post) error {
+
+	// Nothing to do for empty content
+	if post.PostDescription == post.Content &&
+		post.Content == "" {
+		return nil
+	}
+
+	// Same content in both
+	if post.PostDescription == post.Content {
+		post.PostDescription = ""
+
+	}
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(post.Content))
+	if err != nil {
+		return err
+	}
+
+	sel := doc.Find(strings.Join(RemoveSelectors, ","))
+	sel.Remove()
+
+	post.Content, err = doc.Html()
+
+	return err
+}
+
+func extractPreviewText(feed feeds.Feed, post *posts.Post) error {
+	// Ignore filled description
+	if post.PostDescription != "" {
+		return nil