refactor most of file reading/caching code to use one global cache

Signed-off-by: kim (grufwub) <grufwub@gmail.com>
master
kim (grufwub) 4 years ago
parent 51b0b43398
commit 7b834eb479

@ -7,30 +7,17 @@ import (
"container/list"
)
const (
BytesInMegaByte = 1048576.0
)
var (
FileMonitorSleepTime = time.Duration(*CacheCheckFreq) * time.Second
/* Global file caches */
GophermapCache *FileCache
RegularCache *FileCache
GlobalFileCache *FileCache
)
func startFileCaching() {
/* Create gophermap file cache */
GophermapCache = new(FileCache)
GophermapCache.Init(*CacheSize, func(path string) File {
return NewGophermapFile(path)
})
/* Create regular file cache */
RegularCache = new(FileCache)
RegularCache.Init(*CacheSize, func(path string) File {
return NewRegularFile(path)
})
GlobalFileCache = new(FileCache)
GlobalFileCache.Init(*CacheSize, )
/* Start file monitor in separate goroutine */
go startFileMonitor()
@ -42,11 +29,8 @@ func startFileMonitor() {
/* Sleep so we don't take up all the precious CPU time :) */
time.Sleep(FileMonitorSleepTime)
/* Check regular cache freshness */
checkCacheFreshness(RegularCache)
/* Check gophermap cache freshness */
checkCacheFreshness(GophermapCache)
/* Check global file cache freshness */
checkCacheFreshness(GlobalFileCache)
}
/* We shouldn't have reached here */
@ -85,25 +69,8 @@ func checkCacheFreshness(cache *FileCache) {
cache.CacheMutex.RUnlock()
}
type File interface {
/* File contents */
Contents() []byte
LoadContents() *GophorError
/* Cache state */
IsFresh() bool
SetUnfresh()
LastRefresh() int64
/* Mutex */
Lock()
Unlock()
RLock()
RUnlock()
}
type FileElement struct {
File File
File *File
Element *list.Element
}
@ -114,21 +81,34 @@ type FileCache struct {
FileList *list.List
ListMutex sync.Mutex
Size int
NewFile func(path string) File
}
func (fc *FileCache) Init(size int, newFileFunc func(path string) File) {
func (fc *FileCache) Init(size int) {
fc.CacheMap = make(map[string]*FileElement)
fc.CacheMutex = sync.RWMutex{}
fc.FileList = list.New()
fc.FileList.Init()
fc.ListMutex = sync.Mutex{}
fc.Size = size
fc.NewFile = newFileFunc
}
func (fc *FileCache) Fetch(path string) ([]byte, *GophorError) {
func (fc *FileCache) FetchRegular(path string) ([]byte, *GophorError) {
return fc.Fetch(path, func(path string) FileContents {
contents := new(RegularFileContents)
contents.path = path
return contents
})
}
func (fc *FileCache) FetchGophermap(path string) ([]byte, *GophorError) {
return fc.Fetch(path, func(path string) FileContents {
contents := new(GophermapContents)
contents.path = path
return contents
})
}
func (fc *FileCache) Fetch(path string, newFileContents func(string) FileContents) ([]byte, *GophorError) {
/* Get cache map read lock then check if file in cache map */
fc.CacheMutex.RLock()
fileElement, ok := fc.CacheMap[path]
@ -164,8 +144,11 @@ func (fc *FileCache) Fetch(path string) ([]byte, *GophorError) {
return nil, &GophorError{ FileStatErr, err }
}
/* Use supplied new file function */
file := fc.NewFile(path)
/* Create new file contents object using supplied function */
contents := newFileContents(path)
/* Create new file wrapper around contents */
file := NewFile(contents)
/* NOTE: file isn't in cache yet so no need to lock file write mutex
* before loading from disk

@ -0,0 +1,82 @@
package main
const (
/* Parsing */
CrLf = "\r\n"
End = "."
LastLine = End+CrLf
Tab = "\t"
MaxUserNameLen = 70 /* RFC 1436 standard */
MaxSelectorLen = 255 /* RFC 1436 standard */
NullHost = "null.host"
NullPort = "1"
SelectorErrorStr = "selector_length_error"
GophermapRenderErrorStr = ""
ReplaceStrHostname = "$hostname"
/* Filesystem */
GophermapFileStr = "gophermap"
/* Misc */
BytesInMegaByte = 1048576.0
)
/*
* Item type characters:
* Collected from RFC 1436 standard, Wikipedia, Go-gopher project
* and Gophernicus project. Those with ALL-CAPS descriptions in
* [square brackets] defined and used by Gophernicus, a popular
* Gopher server.
*/
type ItemType byte
const (
/* RFC 1436 Standard */
TypeFile = ItemType('0') /* Regular file [TEXT] */
TypeDirectory = ItemType('1') /* Directory [MENU] */
TypePhonebook = ItemType('2') /* CSO phone-book server */
TypeError = ItemType('3') /* Error [ERROR] */
TypeMacBinHex = ItemType('4') /* Binhexed macintosh file */
TypeDosBinArchive = ItemType('5') /* DOS bin archive, CLIENT MUST READ UNTIL TCP CLOSE [GZIP] */
TypeUnixFile = ItemType('6') /* Unix uuencoded file */
TypeIndexSearch = ItemType('7') /* Index-search server [QUERY] */
TypeTelnet = ItemType('8') /* Text-based telnet session */
TypeBin = ItemType('9') /* Binary file, CLIENT MUST READ UNTIL TCP CLOSE [BINARY] */
TypeTn3270 = ItemType('T') /* Text-based tn3270 session */
TypeGif = ItemType('g') /* Gif format graphics file [GIF] */
TypeImage = ItemType('I') /* Some kind of image file (client decides how to display) [IMAGE] */
TypeRedundant = ItemType('+') /* Redundant server */
TypeEnd = ItemType('.') /* Indicates LastLine if only this + CrLf */
/* Non-standard - as used by https://github.com/prologic/go-gopher
* (also seen on Wikipedia: https://en.wikipedia.org/wiki/Gopher_%28protocol%29#Item_types)
*/
TypeInfo = ItemType('i') /* Informational message [INFO] */
TypeHtml = ItemType('h') /* HTML document [HTML] */
TypeAudio = ItemType('s') /* Audio file */
TypePng = ItemType('p') /* PNG image */
TypeDoc = ItemType('d') /* Document [DOC] */
/* Non-standard - as used by Gopernicus https://github.com/gophernicus/gophernicus */
TypeMime = ItemType('M') /* [MIME] */
TypeVideo = ItemType(';') /* [VIDEO] */
TypeCalendar = ItemType('c') /* [CALENDAR] */
TypeTitle = ItemType('!') /* [TITLE] */
TypeComment = ItemType('#') /* [COMMENT] */
TypeHiddenFile = ItemType('-') /* [HIDDEN] Hides file from directory listing */
TypeSubGophermap = ItemType('=') /* [EXECUTE] read this file in here */
TypeEndBeginList = ItemType('*') /* If only this + CrLf, indicates last line but then followed by directory list */
/* Default type */
TypeDefault = TypeBin
/* Gophor specific types */
TypeExec = ItemType('$') /* Execute command and insert stdout here */
TypeInfoNotStated = ItemType('z') /* INTERNAL USE. We use this in a switch case, a line never starts with this */
TypeUnknown = ItemType('?') /* INTERNAL USE. We use this in a switch case, a line never starts with this */
)

@ -1,57 +0,0 @@
package main
import (
"os"
"path"
"strings"
)
func listDir(dirPath string, hidden map[string]bool) ([]byte, *GophorError) {
/* Open directory file descriptor */
fd, err := os.Open(dirPath)
if err != nil {
logSystemError("failed to open %s: %s\n", dirPath, err.Error())
return nil, &GophorError{ FileOpenErr, err }
}
/* Open directory stream for reading */
files, err := fd.Readdir(-1)
if err != nil {
logSystemError("failed to enumerate dir %s: %s\n", dirPath, err.Error())
return nil, &GophorError{ DirListErr, err }
}
var entity *DirEntity
dirContents := make([]byte, 0)
/* Walk through directory */
for _, file := range files {
/* Skip dotfiles + gophermap file + requested hidden */
if file.Name()[0] == '.' || strings.HasSuffix(file.Name(), GophermapFileStr) {
continue
} else if _, ok := hidden[file.Name()]; ok {
continue
}
/* Handle file, directory or ignore others */
switch {
case file.Mode() & os.ModeDir != 0:
/* Directory -- create directory listing */
itemPath := path.Join(fd.Name(), file.Name())
entity = newDirEntity(TypeDirectory, file.Name(), itemPath, *ServerHostname, *ServerPort)
dirContents = append(dirContents, entity.Bytes()...)
case file.Mode() & os.ModeType == 0:
/* Regular file -- find item type and creating listing */
itemPath := path.Join(fd.Name(), file.Name())
itemType := getItemType(itemPath)
entity = newDirEntity(itemType, file.Name(), itemPath, *ServerHostname, *ServerPort)
dirContents = append(dirContents, entity.Bytes()...)
default:
/* Ignore */
}
}
return dirContents, nil
}

@ -1,238 +0,0 @@
package main
import (
"strconv"
"strings"
"path/filepath"
)
const (
CrLf = "\r\n"
End = "."
LastLine = End+CrLf
Tab = byte('\t')
/* MaxUserNameLen = 70 RFC 1436 standard */
MaxSelectorLen = 255 /* RFC 1436 standard */
UserNameErr = "! Err: Max UserName len reached"
SelectorErr = "err_max_selector_len_reached"
NullHost = "null.host"
NullPort = "1"
)
type ItemType byte
/*
* Item type characters:
* Collected from RFC 1436 standard, Wikipedia, Go-gopher project
* and Gophernicus project. Those with ALL-CAPS descriptions in
* [square brackets] defined and used by Gophernicus, a popular
* Gopher server.
*/
const (
/* RFC 1436 Standard */
TypeFile = ItemType('0') /* Regular file [TEXT] */
TypeDirectory = ItemType('1') /* Directory [MENU] */
TypePhonebook = ItemType('2') /* CSO phone-book server */
TypeError = ItemType('3') /* Error [ERROR] */
TypeMacBinHex = ItemType('4') /* Binhexed macintosh file */
TypeDosBinArchive = ItemType('5') /* DOS bin archive, CLIENT MUST READ UNTIL TCP CLOSE [GZIP] */
TypeUnixFile = ItemType('6') /* Unix uuencoded file */
TypeIndexSearch = ItemType('7') /* Index-search server [QUERY] */
TypeTelnet = ItemType('8') /* Text-based telnet session */
TypeBin = ItemType('9') /* Binary file, CLIENT MUST READ UNTIL TCP CLOSE [BINARY] */
TypeTn3270 = ItemType('T') /* Text-based tn3270 session */
TypeGif = ItemType('g') /* Gif format graphics file [GIF] */
TypeImage = ItemType('I') /* Some kind of image file (client decides how to display) [IMAGE] */
TypeRedundant = ItemType('+') /* Redundant server */
TypeEnd = ItemType('.') /* Indicates LastLine if only this + CrLf */
/* Non-standard - as used by https://github.com/prologic/go-gopher
* (also seen on Wikipedia: https://en.wikipedia.org/wiki/Gopher_%28protocol%29#Item_types)
*/
TypeInfo = ItemType('i') /* Informational message [INFO] */
TypeHtml = ItemType('h') /* HTML document [HTML] */
TypeAudio = ItemType('s') /* Audio file */
TypePng = ItemType('p') /* PNG image */
TypeDoc = ItemType('d') /* Document [DOC] */
/* Non-standard - as used by Gopernicus https://github.com/gophernicus/gophernicus */
TypeMime = ItemType('M') /* [MIME] */
TypeVideo = ItemType(';') /* [VIDEO] */
TypeCalendar = ItemType('c') /* [CALENDAR] */
TypeTitle = ItemType('!') /* [TITLE] */
TypeComment = ItemType('#') /* [COMMENT] */
TypeHiddenFile = ItemType('-') /* [HIDDEN] Hides file from directory listing */
TypeSubGophermap = ItemType('=') /* [EXECUTE] read this file in here */
TypeEndBeginList = ItemType('*') /* If only this + CrLf, indicates last line but then followed by directory list */
/* Default type */
TypeDefault = TypeFile
/* Gophor specific types */
TypeExec = ItemType('$') /* Execute command and insert stdout here */
TypeInfoNotStated = ItemType('z') /* INTERNAL USE. We use this in a switch case, a line never starts with this */
TypeUnknown = ItemType('?') /* INTERNAL USE. We use this in a switch case, a line never starts with this */
)
/*
* Directory Entity data structure for easier handling
*/
type DirEntity struct {
/* RFC 1436 standard */
Type ItemType
UserName string
Selector string
Host string
Port string
/* Non-standard, proposed Gopher+
* gopher://gopher.floodgap.com:70/0/gopher/tech/gopherplus.txt
*/
Extras string
}
func newDirEntity(t ItemType, name, selector, host string, port int) *DirEntity {
entity := new(DirEntity)
entity.Type = t
/* Truncate username if we hit MaxUserNameLen */
if len(name) > *PageWidth {
name = name[:*PageWidth-4] + "..."
}
entity.UserName = name
/* Truncate selector if we hit MaxSelectorLen */
if len(selector) > MaxSelectorLen {
selector = SelectorErr
}
entity.Selector = selector
entity.Host = host
entity.Port = strconv.Itoa(port)
return entity
}
func (entity *DirEntity) Bytes() []byte {
b := []byte{}
b = append(b, byte(entity.Type))
b = append(b, []byte(entity.UserName)...)
b = append(b, Tab)
b = append(b, []byte(entity.Selector)...)
b = append(b, Tab)
b = append(b, []byte(entity.Host)...)
b = append(b, Tab)
b = append(b, []byte(entity.Port)...)
if entity.Extras != "" {
b = append(b, Tab)
b = append(b, []byte(entity.Extras)...)
}
b = append(b, []byte(CrLf)...)
return b
}
var FileExtensions = map[string]ItemType{
".out": TypeBin,
".a": TypeBin,
".o": TypeBin,
".ko": TypeBin, /* ... Though tbh, kernel extensions?!!! */
".msi": TypeBin,
".exe": TypeBin,
".txt": TypeFile,
".md": TypeFile,
".json": TypeFile,
".xml": TypeFile,
".yaml": TypeFile,
".ocaml": TypeFile,
".s": TypeFile,
".c": TypeFile,
".py": TypeFile,
".h": TypeFile,
".go": TypeFile,
".fs": TypeFile,
".doc": TypeDoc,
".docx": TypeDoc,
".gif": TypeGif,
".jpg": TypeImage,
".jpeg": TypeImage,
".png": TypeImage,
".html": TypeHtml,
".ogg": TypeAudio,
".mp3": TypeAudio,
".wav": TypeAudio,
".mod": TypeAudio,
".it": TypeAudio,
".xm": TypeAudio,
".mid": TypeAudio,
".vgm": TypeAudio,
".mp4": TypeVideo,
".mkv": TypeVideo,
}
func getItemType(name string) ItemType {
extension := strings.ToLower(filepath.Ext(name))
fileType, ok := FileExtensions[extension]
if !ok {
return TypeDefault
}
return fileType
}
func parseLineType(line string) ItemType {
lineLen := len(line)
if lineLen == 0 {
return TypeInfoNotStated
} else if lineLen == 1 {
/* The only accepted types for a length 1 line */
switch ItemType(line[0]) {
case TypeEnd:
return TypeEnd
case TypeEndBeginList:
return TypeEndBeginList
case TypeComment:
return TypeComment
case TypeInfo:
return TypeInfo
case TypeTitle:
return TypeTitle
default:
return TypeUnknown
}
} else if !strings.Contains(line, string(Tab)) {
/* The only accepted types for a line with no tabs */
switch ItemType(line[0]) {
case TypeComment:
return TypeComment
case TypeTitle:
return TypeTitle
case TypeInfo:
return TypeInfo
case TypeHiddenFile:
return TypeHiddenFile
case TypeSubGophermap:
return TypeSubGophermap
case TypeExec:
return TypeExec
default:
return TypeInfoNotStated
}
}
return ItemType(line[0])
}
func createInfoLine(content string) []byte {
return []byte(string(TypeInfo)+content+string(Tab)+NullHost+string(Tab)+NullPort+CrLf)
}

@ -1,108 +1,25 @@
package main
import (
"os"
"io"
"bufio"
"sync"
"time"
)
type RegularFile struct {
path string
contents []byte
mutex sync.RWMutex
isFresh bool
lastRefresh int64
}
func NewRegularFile(path string) *RegularFile {
f := new(RegularFile)
f.path = path
f.mutex = sync.RWMutex{}
return f
/* RegularFileContents:
* Very simple implementation of FileContents that just
* buffered reads from the stored file path, stores the
* read bytes in a slice and returns when requested.
*/
type RegularFileContents struct {
path string
contents []byte
}
func (f *RegularFile) Contents() []byte {
return f.contents
func (fc *RegularFileContents) Render() []byte {
return fc.contents
}
func (f *RegularFile) LoadContents() *GophorError {
/* Clear current cache */
f.contents = nil
/* Reload the file */
func (fc *RegularFileContents) Load() *GophorError {
var gophorErr *GophorError
f.contents, gophorErr = bufferedRead(f.path)
/* Update lastRefresh time + set fresh */
f.lastRefresh = time.Now().UnixNano()
f.isFresh = true
fc.contents, gophorErr = bufferedRead(fc.path)
return gophorErr
}
func (f *RegularFile) IsFresh() bool {
return f.isFresh
}
func (f *RegularFile) SetUnfresh() {
f.isFresh = false
}
func (f *RegularFile) LastRefresh() int64 {
return f.lastRefresh
}
func (f *RegularFile) Lock() {
f.mutex.Lock()
}
func (f *RegularFile) Unlock() {
f.mutex.Unlock()
}
func (f *RegularFile) RLock() {
f.mutex.RLock()
}
func (f *RegularFile) RUnlock() {
f.mutex.RUnlock()
}
func bufferedRead(path string) ([]byte, *GophorError) {
/* Open file */
fd, err := os.Open(path)
if err != nil {
return nil, &GophorError{ FileOpenErr, err }
}
defer fd.Close()
/* Setup buffers */
var count int
contents := make([]byte, 0)
buf := make([]byte, FileReadBufSize)
/* Setup reader */
reader := bufio.NewReader(fd)
/* Read through buffer until error or null bytes! */
for {
count, err = reader.Read(buf)
if err != nil {
if err == io.EOF {
break
}
return nil, &GophorError{ FileReadErr, err }
}
contents = append(contents, buf[:count]...)
if count < FileReadBufSize {
break
}
}
return contents, nil
func (fc *RegularFileContents) Clear() {
fc.contents = nil
}

@ -1,312 +0,0 @@
package main
import (
"os"
"io"
"bufio"
"bytes"
"strings"
"sync"
"time"
)
const (
GophermapFileStr = "gophermap"
ReplaceStringHostname = "$hostname"
)
type GophermapSection interface {
Render() ([]byte, *GophorError)
}
type GophermapText struct {
contents []byte
}
func NewGophermapText(contents string) *GophermapText {
s := new(GophermapText)
s.contents = []byte(contents)
return s
}
func (s *GophermapText) Render() ([]byte, *GophorError) {
return s.contents, nil
}
type GophermapDirListing struct {
path string
Hidden map[string]bool
}
func NewGophermapDirListing(path string) *GophermapDirListing {
s := new(GophermapDirListing)
s.path = path
return s
}
func (s *GophermapDirListing) Render() ([]byte, *GophorError) {
return listDir(s.path, s.Hidden)
}
type GophermapFile struct {
path string
lines []GophermapSection
mutex sync.RWMutex
isFresh bool
lastRefresh int64
}
func NewGophermapFile(path string) *GophermapFile {
f := new(GophermapFile)
f.path = path
f.mutex = sync.RWMutex{}
return f
}
func (f *GophermapFile) Contents() []byte {
/* We don't just want to read the contents,
* but also execute any included gophermap
* execute lines.
*/
contents := make([]byte, 0)
for _, line := range f.lines {
content, gophorErr := line.Render()
if gophorErr != nil {
content = []byte(string(TypeInfo)+"Error rendering gophermap section."+CrLf)
}
contents = append(contents, content...)
}
return contents
}
func (f *GophermapFile) LoadContents() *GophorError {
/* Clear the current cache */
f.lines = nil
/* Reload the file */
var gophorErr *GophorError
f.lines, gophorErr = f.readGophermap(f.path)
/* Update lastRefresh + set fresh */
f.lastRefresh = time.Now().UnixNano()
f.isFresh = true
return gophorErr
}
func (f *GophermapFile) IsFresh() bool {
return f.isFresh
}
func (f *GophermapFile) SetUnfresh() {
f.isFresh = false
}
func (f *GophermapFile) LastRefresh() int64 {
return f.lastRefresh
}
func (f *GophermapFile) Lock() {
f.mutex.Lock()
}
func (f *GophermapFile) Unlock() {
f.mutex.Unlock()
}
func (f *GophermapFile) RLock() {
f.mutex.RLock()
}
func (f *GophermapFile) RUnlock() {
f.mutex.RUnlock()
}
func (f *GophermapFile) readGophermap(path string) ([]GophermapSection, *GophorError) {
/* First, read raw file contents */
contents, gophorErr := bufferedRead(path)
if gophorErr != nil {
return nil, gophorErr
}
/* Create reader and scanner from this */
reader := bytes.NewReader(contents)
scanner := bufio.NewScanner(reader)
/* Setup scanner to split on CrLf */
scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
/* At EOF, no more data */
return 0, nil, nil
}
if i := bytes.Index(data, []byte{ '\r', '\n' }); i >= 0 {
/* We have a full new-line terminate line */
return i+2, data[0:i], nil
}
/* Request more data */
return 0, nil, nil
})
/* Create return slice + hidden files map in case dir listing requested */
sections := make([]GophermapSection, 0)
hidden := make(map[string]bool)
var dirListing *GophermapDirListing
/* TODO: work on efficiency */
/* Scan, format each token and add to parsedContents */
doEnd := false
for scanner.Scan() {
line := scanner.Text()
/* Parse the line item type and handle */
lineType := parseLineType(line)
switch lineType {
case TypeInfoNotStated:
/* Append TypeInfo to the beginning of line */
sections = append(sections, NewGophermapText(string(createInfoLine(line))))
case TypeComment:
/* We ignore this line */
continue
case TypeHiddenFile:
/* Add to hidden files map */
hidden[line[1:]] = true
case TypeSubGophermap:
/* Check if we've been supplied subgophermap or regular file */
if strings.HasSuffix(line[1:], GophermapFileStr) {
/* Ensure we haven't been passed the current gophermap. Recursion bad! */
if line[1:] == path {
continue
}
/* Treat as any other gopher map! */
submapSections, gophorErr := f.readGophermap(line[1:])
if gophorErr != nil {
/* Failed to read subgophermap, insert error line */
sections = append(sections, NewGophermapText(string(TypeInfo)+"Error reading subgophermap: "+line[1:]+CrLf))
} else {
sections = append(sections, submapSections...)
}
} else {
/* Treat as regular file, but we need to replace Unix line endings
* with gophermap line endings
*/
fileContents, gophorErr := bufferedReadAsGophermap(line[1:])
if gophorErr != nil {
/* Failed to read file, insert error line */
sections = append(sections, NewGophermapText(string(TypeInfo)+"Error reading subgophermap: "+line[1:]+CrLf))
} else {
sections = append(sections, NewGophermapText(string(fileContents)))
}
}
case TypeExec:
/* Try executing supplied line */
sections = append(sections, NewGophermapText(string(TypeInfo)+"Error: inline shell commands not yet supported"+CrLf))
case TypeEnd:
/* Lastline, break out at end of loop. Interface method Contents()
* will append a last line at the end so we don't have to worry about
* that here, only stopping the loop.
*/
doEnd = true
case TypeEndBeginList:
/* Create GophermapDirListing object then break out at end of loop */
doEnd = true
dirListing = NewGophermapDirListing(strings.TrimSuffix(path, GophermapFileStr))
default:
/* Replace pre-set strings */
line = strings.Replace(line, ReplaceStringHostname, *ServerHostname, -1)
sections = append(sections, NewGophermapText(line+CrLf))
}
/* Break out of read loop if requested */
if doEnd {
break
}
}
/* If scanner didn't finish cleanly, return nil and error */
if scanner.Err() != nil {
return nil, &GophorError{ FileReadErr, scanner.Err() }
}
/* If dir listing requested, append the hidden files map then add
* to sections slice. We can do this here as the TypeEndBeginList item
* type ALWAYS comes last, at least in the gophermap handled by this context.
*/
if dirListing != nil {
dirListing.Hidden = hidden
sections = append(sections, dirListing)
}
return sections, nil
}
func bufferedReadAsGophermap(path string) ([]byte, *GophorError) {
/* Open file */
fd, err := os.Open(path)
if err != nil {
logSystemError("failed to open %s: %s\n", path, err.Error())
return nil, &GophorError{ FileOpenErr, err }
}
defer fd.Close()
/* Create buffered reader from file descriptor */
reader := bufio.NewReader(fd)
fileContents := make([]byte, 0)
/* TODO: work on efficiency */
for {
/* Read up to each new-line */
str, err := reader.ReadString('\n')
if err != nil {
if err == io.EOF {
/* Reached EOF */
break
}
return nil, &GophorError{ FileReadErr, nil }
}
/* Replace single newline with as such */
if str == "\n" {
fileContents = append(fileContents, createInfoLine("")...)
continue
}
/* Replace the newline character */
str = strings.Replace(str, "\n", "", -1)
/* Iterate through returned str, reflowing to new line
* until all lines < PageWidth
*/
for len(str) > 0 {
length := minWidth(len(str))
fileContents = append(fileContents, createInfoLine(str[:length])...)
str = str[length:]
}
}
if !bytes.HasSuffix(fileContents, []byte(CrLf)) {
fileContents = append(fileContents, []byte(CrLf)...)
}
return fileContents, nil
}
func minWidth(w int) int {
if w <= *PageWidth {
return w
} else {
return *PageWidth
}
}

@ -0,0 +1,143 @@
package main
import (
"strconv"
"strings"
"path/filepath"
)
var FileExtensions = map[string]ItemType{
".out": TypeBin,
".a": TypeBin,
".o": TypeBin,
".ko": TypeBin, /* ... Though tbh, kernel extensions?!!! */
".msi": TypeBin,
".exe": TypeBin,
".txt": TypeFile,
".md": TypeFile,
".json": TypeFile,
".xml": TypeFile,
".yaml": TypeFile,
".ocaml": TypeFile,
".s": TypeFile,
".c": TypeFile,
".py": TypeFile,
".h": TypeFile,
".go": TypeFile,
".fs": TypeFile,
".doc": TypeDoc,
".docx": TypeDoc,
".gif": TypeGif,
".jpg": TypeImage,
".jpeg": TypeImage,
".png": TypeImage,
".html": TypeHtml,
".ogg": TypeAudio,
".mp3": TypeAudio,
".wav": TypeAudio,
".mod": TypeAudio,
".it": TypeAudio,
".xm": TypeAudio,
".mid": TypeAudio,
".vgm": TypeAudio,
".mp4": TypeVideo,
".mkv": TypeVideo,
}
func buildLine(t ItemType, name, selector, host string, port int) []byte {
ret := string(t)
/* Add name, truncate name if too long */
if len(name) > *PageWidth {
ret += name[:*PageWidth-4]+"...\t"
} else {
ret += name+"\t"
}
/* Add selector. If too long use err, skip if empty */
selectorLen := len(selector)
if selectorLen > MaxSelectorLen {
ret += SelectorErrorStr+"\t"
} else if selectorLen > 0 {
ret += selector+"\t"
}
/* Add host, set to nullhost if empty */
if host == "" {
ret += NullHost+"\t"
}
/* Add port, set to nullport if 0 */
if port == 0 {
ret += NullPort+CrLf
} else {
ret += strconv.Itoa(port)+CrLf
}
return []byte(ret)
}
func buildInfoLine(content string) []byte {
return buildLine(TypeInfo, content, "", "", 0)
}
func getItemType(name string) ItemType {
extension := strings.ToLower(filepath.Ext(name))
fileType, ok := FileExtensions[extension]
if !ok {
return TypeDefault
}
return fileType
}
func parseLineType(line string) ItemType {
lineLen := len(line)
if lineLen == 0 {
return TypeInfoNotStated
} else if lineLen == 1 {
/* The only accepted types for a length 1 line */
switch ItemType(line[0]) {
case TypeEnd:
return TypeEnd
case TypeEndBeginList:
return TypeEndBeginList
case TypeComment:
return TypeComment
case TypeInfo:
return TypeInfo
case TypeTitle:
return TypeTitle
default:
return TypeUnknown
}
} else if !strings.Contains(line, string(Tab)) {
/* The only accepted types for a line with no tabs */
switch ItemType(line[0]) {
case TypeComment:
return TypeComment
case TypeTitle:
return TypeTitle
case TypeInfo:
return TypeInfo
case TypeHiddenFile:
return TypeHiddenFile
case TypeSubGophermap:
return TypeSubGophermap
case TypeExec:
return TypeExec
default:
return TypeInfoNotStated
}
}
return ItemType(line[0])
}

208
fs.go

@ -0,0 +1,208 @@
package main
import (
"os"
"sync"
"path"
"strings"
"bytes"
"time"
"io"
"bufio"
)
/* File:
* Wraps around the cached contents of a file and
* helps with management of this content by the
* global FileCache objects.
*/
type File struct {
contents FileContents
mutex sync.RWMutex
isFresh bool
lastRefresh int64
}
func NewFile(contents FileContents) *File {
f := new(File)
f.contents = contents
f.mutex = sync.RWMutex{}
f.isFresh = true
f.lastRefresh = 0
return f
}
func (f *File) Contents() []byte {
return f.contents.Render()
}
func (f *File) LoadContents() *GophorError {
/* Clear current file contents */
f.contents.Clear()
/* Reload the file */
gophorErr := f.contents.Load()
if gophorErr != nil {
return gophorErr
}
/* Update lastRefresh + set fresh */
f.lastRefresh = time.Now().UnixNano()
f.isFresh = true
return nil
}
func (f *File) IsFresh() bool {
return f.isFresh
}
func (f *File) SetUnfresh() {
f.isFresh = false
}
func (f *File) LastRefresh() int64 {
return f.lastRefresh
}
func (f *File) Lock() {
f.mutex.Lock()
}
func (f *File) Unlock() {
f.mutex.Unlock()
}
func (f *File) RLock() {
f.mutex.RLock()
}
func (f *File) RUnlock() {
f.mutex.RUnlock()
}
/* FileContents:
* Interface that provides an adaptable implementation
* for holding onto some level of information about
* the contents of a file, also methods for processing
* and returning the results when the file contents
* are requested.
*/
type FileContents interface {
Render() []byte
Load() *GophorError
Clear()
}
func bufferedRead(path string) ([]byte, *GophorError) {
/* Open file */
fd, err := os.Open(path)
if err != nil {
return nil, &GophorError{ FileOpenErr, err }
}
defer fd.Close()
/* Setup buffers */
var count int
contents := make([]byte, 0)
buf := make([]byte, FileReadBufSize)
/* Setup reader */
reader := bufio.NewReader(fd)
/* Read through buffer until error or null bytes! */
for {
count, err = reader.Read(buf)
if err != nil {
if err == io.EOF {
break
}
return nil, &GophorError{ FileReadErr, err }
}
contents = append(contents, buf[:count]...)
if count < FileReadBufSize {
break
}
}
return contents, nil
}
func bufferedScan(path string,
scanSplitter func([]byte, bool) (int, []byte, error),
scanIterator func(*bufio.Scanner) bool) *GophorError {
/* First, read raw file contents */
contents, gophorErr := bufferedRead(path)
if gophorErr != nil {
return gophorErr
}
/* Create reader and scanner from this */
reader := bytes.NewReader(contents)
scanner := bufio.NewScanner(reader)
/* Setup scanner splitter */
scanner.Split(scanSplitter)
/* Scan through file contents using supplied iterator */
for scanner.Scan() && scanIterator(scanner) {}
/* Check scanner finished cleanly */
if scanner.Err() != nil {
return &GophorError{ FileReadErr, scanner.Err() }
}
return nil
}
func listDir(dirPath string, hidden map[string]bool) ([]byte, *GophorError) {
/* Open directory file descriptor */
fd, err := os.Open(dirPath)
if err != nil {
logSystemError("failed to open %s: %s\n", dirPath, err.Error())
return nil, &GophorError{ FileOpenErr, err }
}
/* Open directory stream for reading */
files, err := fd.Readdir(-1)
if err != nil {
logSystemError("failed to enumerate dir %s: %s\n", dirPath, err.Error())
return nil, &GophorError{ DirListErr, err }
}
dirContents := make([]byte, 0)
/* Walk through directory */
for _, file := range files {
/* Skip dotfiles + gophermap file + requested hidden */
if file.Name()[0] == '.' || strings.HasSuffix(file.Name(), GophermapFileStr) {
continue
} else if _, ok := hidden[file.Name()]; ok {
continue
}
/* Handle file, directory or ignore others */
switch {
case file.Mode() & os.ModeDir != 0:
/* Directory -- create directory listing */
itemPath := path.Join(fd.Name(), file.Name())
line := buildLine(TypeDirectory, file.Name(), itemPath, *ServerHostname, *ServerPort)
dirContents = append(dirContents, line...)
case file.Mode() & os.ModeType == 0:
/* Regular file -- find item type and creating listing */
itemPath := path.Join(fd.Name(), file.Name())
itemType := getItemType(itemPath)
line := buildLine(itemType, file.Name(), itemPath, *ServerHostname, *ServerPort)
dirContents = append(dirContents, line...)
default:
/* Ignore */
}
}
return dirContents, nil
}

@ -0,0 +1,271 @@
package main
import (
"bytes"
"bufio"
"strings"
)
/* GophermapContents:
* Implementation of FileContents that reads and
* parses a gophermap file into a slice of gophermap
* sections, then renders and returns these sections
* when requested.
*/
type GophermapContents struct {
path string
sections []GophermapSection
}
func (gc *GophermapContents) Render() []byte {
/* We don't just want to read the contents, but also
* execute any included gophermap execute lines
*/
returnContents := make([]byte, 0)
for _, line := range gc.sections {
content, gophorErr := line.Render()
if gophorErr != nil {
content = buildInfoLine(GophermapRenderErrorStr)
}
returnContents = append(returnContents, content...)
}
return returnContents
}
func (gc *GophermapContents) Load() *GophorError {
var gophorErr *GophorError
gc.sections, gophorErr = readGophermap(gc.path)
return gophorErr
}
func (gc *GophermapContents) Clear() {
gc.sections = nil
}
/* GophermapSection:
* Provides an interface for different stored sections
* of a gophermap file, whether it's static text that we
* may want stored as-is, or the data required for a dir
* listing or command executed that we may want updated
* upon each file cache request.
*/
type GophermapSection interface {
Render() ([]byte, *GophorError)
}
/* GophermapText:
* Simple implementation of GophermapSection that holds
* onto a static section of text as a slice of bytes.
*/
type GophermapText struct {
contents []byte
}
func NewGophermapText(contents []byte) *GophermapText {
s := new(GophermapText)
s.contents = contents
return s
}
func (s *GophermapText) Render() ([]byte, *GophorError) {
return s.contents, nil
}
/* GophermapDirListing:
* An implementation of GophermapSection that holds onto a
* path and a requested list of hidden files, then enumerates
* the supplied paths (ignoring hidden files) when the content
* Render() call is received.
*/
type GophermapDirListing struct {
path string
Hidden map[string]bool
}
func NewGophermapDirListing(path string) *GophermapDirListing {
s := new(GophermapDirListing)
s.path = path
return s
}
func (s *GophermapDirListing) Render() ([]byte, *GophorError) {
return listDir(s.path, s.Hidden)
}
func readGophermap(path string) ([]GophermapSection, *GophorError) {
/* Create return slice. Also hidden files map in case dir listing requested */
sections := make([]GophermapSection, 0)
hidden := make(map[string]bool)
var dirListing *GophermapDirListing
/* Perform buffered scan with our supplied splitter and iterators */
gophorErr := bufferedScan(path,
func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
/* At EOF, no more data */
return 0, nil, nil
}
if i := bytes.Index(data, []byte("\r\n")); i >= 0 {
/* We have a full new-line terminate line */
return i+2, data[:i], nil
}
/* Request more data */
return 0, nil, nil
},
func(scanner *bufio.Scanner) bool {
line := scanner.Text()
/* Parse the line item type and handle */
lineType := parseLineType(line)
switch lineType {
case TypeInfoNotStated:
/* Append TypeInfo to the beginning of line */
sections = append(sections, NewGophermapText(buildInfoLine(line)))
case TypeComment:
/* We ignore this line */
break
case TypeHiddenFile:
/* Add to hidden files map */
hidden[line[1:]] = true
case TypeSubGophermap:
/* Check if we've been supplied subgophermap or regular file */
if strings.HasSuffix(line[1:], GophermapFileStr) {
/* Ensure we haven't been passed the current gophermap. Recursion bad! */
if line[1:] == path {
break
}
/* Treat as any other gopher map! */
submapSections, gophorErr := readGophermap(line[1:])
if gophorErr != nil {
/* Failed to read subgophermap, insert error line */
sections = append(sections, NewGophermapText(buildInfoLine("Error reading subgophermap: "+line[1:])))
} else {
sections = append(sections, submapSections...)
}
} else {
/* Treat as regular file, but we need to replace Unix line endings
* with gophermap line endings
*/
fileContents, gophorErr := readIntoGophermap(line[1:])
if gophorErr != nil {
/* Failed to read file, insert error line */
sections = append(sections, NewGophermapText(buildInfoLine("Error reading subgophermap: "+line[1:])))
} else {
sections = append(sections, NewGophermapText(fileContents))
}
}
case TypeExec:
/* Try executing supplied line */
sections = append(sections, NewGophermapText(buildInfoLine("Error: inline shell commands not yet supported")))
case TypeEnd:
/* Lastline, break out at end of loop. Interface method Contents()
* will append a last line at the end so we don't have to worry about
* that here, only stopping the loop.
*/
return false
case TypeEndBeginList:
/* Create GophermapDirListing object then break out at end of loop */
dirListing = NewGophermapDirListing(strings.TrimSuffix(path, GophermapFileStr))
return false
default:
/* Replace pre-set strings */
line = strings.Replace(line, ReplaceStrHostname, *ServerHostname, -1)
sections = append(sections, NewGophermapText([]byte(line+CrLf)))
}
return true
},
)
/* Check the bufferedScan didn't exit with error */
if gophorErr != nil {
return nil, gophorErr
}
/* If dir listing requested, append the hidden files map then add
* to sections slice. We can do this here as the TypeEndBeginList item
* type ALWAYS comes last, at least in the gophermap handled by this context.
*/
if dirListing != nil {
dirListing.Hidden = hidden
sections = append(sections, dirListing)
}
return sections, nil
}
func readIntoGophermap(path string) ([]byte, *GophorError) {
/* Create return slice */
fileContents := make([]byte, 0)
/* Perform buffered scan with our supplied splitter and iterators */
gophorErr := bufferedScan(path,
func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
/* At EOF, no more data */
return 0, nil, nil
}
if i := bytes.Index(data, []byte("\n")); i >= 0 {
/* We have a full new-line terminate line */
return i+1, data[:i+1], nil
}
/* Request more data */
return 0, nil, nil
},
func(scanner *bufio.Scanner) bool {
line := scanner.Text()
if line == "\n" {
fileContents = append(fileContents, buildInfoLine("")...)
return true
}
/* Replace the newline character */
line = strings.Replace(line, "\n", "", -1)
/* Iterate through returned str, reflowing to new line
* until all lines < PageWidth
*/
for len(line) > 0 {
length := minWidth(len(line))
fileContents = append(fileContents, buildInfoLine(line[:length])...)
line = line[length:]
}
return true
},
)
/* Check the bufferedScan didn't exit with error */
if gophorErr != nil {
return nil, gophorErr
}
/* Check final output ends on a newline */
if !bytes.HasSuffix(fileContents, []byte(CrLf)) {
fileContents = append(fileContents, []byte(CrLf)...)
}
return fileContents, nil
}
func minWidth(w int) int {
if w <= *PageWidth {
return w
} else {
return *PageWidth
}
}

@ -122,7 +122,7 @@ func (worker *Worker) Respond(data []byte) *GophorError {
dataStr := ""
dataLen := len(data)
for i := 0; i < dataLen; i += 1 {
if data[i] == Tab {
if data[i] == '\t' {
break
} else if data[i] == CrLf[0] {
if i == dataLen-1 {
@ -184,7 +184,7 @@ func (worker *Worker) Respond(data []byte) *GophorError {
case Dir:
/* First try to serve gopher map */
gophermapPath := path.Join(requestPath, "/"+GophermapFileStr)
fileContents, gophorErr := GophermapCache.Fetch(gophermapPath)
fileContents, gophorErr := GlobalFileCache.FetchGophermap(gophermapPath)
if gophorErr != nil {
/* Get directory listing instead */
fileContents, gophorErr = listDir(requestPath, map[string]bool{})
@ -208,7 +208,7 @@ func (worker *Worker) Respond(data []byte) *GophorError {
/* Regular file */
case File:
/* Read file contents */
fileContents, gophorErr := RegularCache.Fetch(requestPath)
fileContents, gophorErr := GlobalFileCache.FetchRegular(requestPath)
if gophorErr != nil {
worker.SendErrorText("file read fail\n")
return gophorErr

Loading…
Cancel
Save