Archives
/
gophi
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

major rework to worker response logic

Browse Source 
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
master
kim (grufwub) 4 years ago
parent 47a30f1b23
commit 06eca4d50a
7 changed files with 180 additions and 210 deletions
Show Stats Download Patch File Download Diff File

131
cache.go
Unescape Escape View File

@ -4,6 +4,8 @@ import (
"os"
"sync"
"time"
"path"
"strings"
)
func startFileMonitor(sleepTime time.Duration) {
@ -23,12 +25,12 @@ func startFileMonitor(sleepTime time.Duration) {
func checkCacheFreshness() {
/* Before anything, get cache write lock (in case we have to delete) */
Config.FileCache.CacheMutex.Lock()
Config.FileSystem.CacheMutex.Lock()
/* Iterate through paths in cache map to query file last modified times */
for path := range Config.FileCache.CacheMap.Map {
for path := range Config.FileSystem.CacheMap.Map {
/* Get file pointer, no need for lock as we have write lock */
file := Config.FileCache.CacheMap.Get(path)
file := Config.FileSystem.CacheMap.Get(path)
/* If this is a generated file, we skip */
if isGeneratedType(file) {
@ -39,7 +41,7 @@ func checkCacheFreshness() {
if err != nil {
/* Log file as not in cache, then delete */
Config.LogSystemError("Failed to stat file in cache: %s\n", path)
Config.FileCache.CacheMap.Remove(path)
Config.FileSystem.CacheMap.Remove(path)
continue
}
timeModified := stat.ModTime().UnixNano()
@ -51,7 +53,7 @@ func checkCacheFreshness() {
}
/* Done! We can release cache read lock */
Config.FileCache.CacheMutex.Unlock()
Config.FileSystem.CacheMutex.Unlock()
}
func isGeneratedType(file *File) bool {
@ -69,44 +71,78 @@ func isGeneratedType(file *File) bool {
* to remove cached files in a LRU style. Uses a RW mutex to lock the
* cache map for appropriate functions and ensure thread safety.
*/
type FileCache struct {
CacheMap *FixedMap
CacheMutex sync.RWMutex
FileSizeMax int64
type FileSystem struct {
CacheMap *FixedMap
CacheMutex sync.RWMutex
CacheFileMax int64
}
func (fc *FileCache) Init(size int, fileSizeMax float64) {
fc.CacheMap = NewFixedMap(size)
fc.CacheMutex = sync.RWMutex{}
fc.FileSizeMax = int64(BytesInMegaByte * fileSizeMax)
func (fs *FileSystem) Init(size int, fileSizeMax float64) {
fs.CacheMap = NewFixedMap(size)
fs.CacheMutex = sync.RWMutex{}
fs.CacheFileMax = int64(BytesInMegaByte * fileSizeMax)
}
func (fc *FileCache) FetchRegular(request *FileSystemRequest) ([]byte, *GophorError) {
/* Calls fc.Fetch() but with the filecontents init function for a regular file */
return fc.Fetch(request, func(path string) FileContents {
return &RegularFileContents{
path,
nil,
func (fs *FileSystem) HandleRequest(requestPath string, host *ConnHost) ([]byte, *GophorError) {
/* Stat filesystem for request file type */
fileType := FileTypeDir;
if requestPath != "." {
stat, err := os.Stat(requestPath)
if err != nil {
/* Check file isn't in cache before throwing in the towel */
fs.CacheMutex.RLock()
file := fs.CacheMap.Get(requestPath)
if file == nil {
fs.CacheMutex.RUnlock()
return nil, &GophorError{ FileStatErr, err }
}
/* It's there! Get contents, unlock and return */
file.Mutex.RLock()
b := file.Contents(&FileSystemRequest{ requestPath, host })
file.Mutex.RUnlock()
fs.CacheMutex.RUnlock()
return b, nil
}
})
}
func (fc *FileCache) FetchGophermap(request *FileSystemRequest) ([]byte, *GophorError) {
/* Calls fc.Fetch() but with the filecontents init function for a gophermap */
return fc.Fetch(request, func(path string) FileContents {
return &GophermapContents{
path,
nil,
/* Set file type for later handling */
switch {
case stat.Mode() & os.ModeDir != 0:
/* do nothing, already set :) */
case stat.Mode() & os.ModeType == 0:
fileType = FileTypeRegular
default:
fileType = FileTypeBad
}
})
}
switch fileType {
/* Directory */
case FileTypeDir:
_, err := os.Stat(path.Join(requestPath, GophermapFileStr))
if err == nil {
/* Gophermap exists, serve this! */
return fs.FetchFile(&FileSystemRequest{ requestPath, host })
} else {
/* No gophermap, serve directory listing */
return listDir(&FileSystemRequest{ requestPath, host }, map[string]bool{})
}
/* Regular file */
case FileTypeRegular:
return fs.FetchFile(&FileSystemRequest{ requestPath, host })
/* Unsupported type */
default:
return nil, &GophorError{ FileTypeErr, nil }
}
}
func (fc *FileCache) Fetch(request *FileSystemRequest, newFileContents func(string) FileContents) ([]byte, *GophorError) {
func (fs *FileSystem) FetchFile(request *FileSystemRequest) ([]byte, *GophorError) {
/* Get cache map read lock then check if file in cache map */
fc.CacheMutex.RLock()
file := fc.CacheMap.Get(request.Path)
/* TODO: work on efficiency. improve use of mutex?? */
fs.CacheMutex.RLock()
file := fs.CacheMap.Get(request.Path)
if file != nil {
/* File in cache -- before doing anything get file read lock */
@ -123,7 +159,7 @@ func (fc *FileCache) Fetch(request *FileSystemRequest, newFileContents func(stri
if gophorErr != nil {
/* Error loading contents, unlock all mutex then return error */
file.Mutex.Unlock()
fc.CacheMutex.RUnlock()
fs.CacheMutex.RUnlock()
return nil, gophorErr
}
@ -138,12 +174,17 @@ func (fc *FileCache) Fetch(request *FileSystemRequest, newFileContents func(stri
stat, err := os.Stat(request.Path)
if err != nil {
/* Error stat'ing file, unlock read mutex then return error */
fc.CacheMutex.RUnlock()
fs.CacheMutex.RUnlock()
return nil, &GophorError{ FileStatErr, err }
}
/* Create new file contents object using supplied function */
contents := newFileContents(request.Path)
var contents FileContents
if strings.HasSuffix(request.Path, "/"+GophermapFileStr) {
contents = &GophermapContents{ request.Path, nil }
} else {
contents = &RegularFileContents{ request.Path, nil }
}
/* Create new file wrapper around contents */
file = NewFile(contents)
@ -152,32 +193,32 @@ func (fc *FileCache) Fetch(request *FileSystemRequest, newFileContents func(stri
gophorErr := file.LoadContents()
if gophorErr != nil {
/* Error loading contents, unlock read mutex then return error */
fc.CacheMutex.RUnlock()
fs.CacheMutex.RUnlock()
return nil, gophorErr
}
/* Compare file size (in MB) to CacheFileSizeMax, if larger just get file
* contents, unlock all mutex and don't bother caching.
*/
if stat.Size() > fc.FileSizeMax {
if stat.Size() > fs.CacheFileMax {
b := file.Contents(request)
fc.CacheMutex.RUnlock()
fs.CacheMutex.RUnlock()
return b, nil
}
/* File not in cache -- Swap cache map read for write lock. */
fc.CacheMutex.RUnlock()
fc.CacheMutex.Lock()
fs.CacheMutex.RUnlock()
fs.CacheMutex.Lock()
/* Put file in the FixedMap */
fc.CacheMap.Put(request.Path, file)
fs.CacheMap.Put(request.Path, file)
/* Before unlocking cache mutex, lock file read for upcoming call to .Contents() */
file.Mutex.RLock()
/* Swap cache lock back to read */
fc.CacheMutex.Unlock()
fc.CacheMutex.RLock()
fs.CacheMutex.Unlock()
fs.CacheMutex.RLock()
}
/* Read file contents into new variable for return, then unlock file read lock */
@ -185,7 +226,7 @@ func (fc *FileCache) Fetch(request *FileSystemRequest, newFileContents func(stri
file.Mutex.RUnlock()
/* Finally we can unlock the cache map read lock, we are done :) */
fc.CacheMutex.RUnlock()
fs.CacheMutex.RUnlock()
return b, nil
}

7
config.go
Unescape Escape View File

@ -15,11 +15,6 @@ type ServerConfig struct {
/* Base settings */
RootDir string
/* Caps.txt information */
Description string
AdminEmail string
Geolocation string
/* Content settings */
PageWidth int
RestrictedFiles []*regexp.Regexp
@ -29,7 +24,7 @@ type ServerConfig struct {
AccessLogger *log.Logger
/* Cache */
FileCache *FileCache
FileSystem *FileSystem
}
func (config *ServerConfig) LogSystem(fmt string, args ...interface{}) {

131
format.go
Unescape Escape View File

@ -5,65 +5,78 @@ import (
)
var FileExtMap = map[string]ItemType{
".out": TypeBin,
".a": TypeBin,
".o": TypeBin,
".ko": TypeBin, /* ... Though tbh, kernel extensions?!!! */
".msi": TypeBin,
".exe": TypeBin,
".lz": TypeBinArchive,
".gz": TypeBinArchive,
".bz2": TypeBinArchive,
".7z": TypeBinArchive,
".zip": TypeBinArchive,
".gitignore": TypeFile,
".txt": TypeFile,
".json": TypeFile,
".yaml": TypeFile,
".ocaml": TypeFile,
".s": TypeFile,
".c": TypeFile,
".py": TypeFile,
".h": TypeFile,
".go": TypeFile,
".fs": TypeFile,
".odin": TypeFile,
".vim": TypeFile,
".nanorc": TypeFile,
".md": TypeMarkup,
".xml": TypeXml,
".doc": TypeDoc,
".docx": TypeDoc,
".pdf": TypeDoc,
".jpg": TypeImage,
".jpeg": TypeImage,
".png": TypeImage,
".gif": TypeImage,
".html": TypeHtml,
".htm": TypeHtml,
".ogg": TypeAudio,
".mp3": TypeAudio,
".wav": TypeAudio,
".mod": TypeAudio,
".it": TypeAudio,
".xm": TypeAudio,
".mid": TypeAudio,
".vgm": TypeAudio,
".opus": TypeAudio,
".m4a": TypeAudio,
".aac": TypeAudio,
".mp4": TypeVideo,
".mkv": TypeVideo,
".webm": TypeVideo,
".out": TypeBin,
".a": TypeBin,
".o": TypeBin,
".ko": TypeBin, /* ... Though tbh, kernel extensions?!!! */
".msi": TypeBin,
".exe": TypeBin,
".lz": TypeBinArchive,
".gz": TypeBinArchive,
".bz2": TypeBinArchive,
".7z": TypeBinArchive,
".zip": TypeBinArchive,
".gitignore": TypeFile,
".txt": TypeFile,
".json": TypeFile,
".yaml": TypeFile,
".ocaml": TypeFile,
".s": TypeFile,
".c": TypeFile,
".py": TypeFile,
".h": TypeFile,
".go": TypeFile,
".fs": TypeFile,
".odin": TypeFile,
".nanorc": TypeFile,
".bashrc": TypeFile,
".mkshrc": TypeFile,
".vimrc": TypeFile,
".vim": TypeFile,
".viminfo": TypeFile,
".sh": TypeFile,
".conf": TypeFile,
".xinitrc": TypeFile,
".jstarrc": TypeFile,
".joerc": TypeFile,
".jpicorc": TypeFile,
".profile": TypeFile,
".bash_profile": TypeFile,
".bash_logout": TypeFile,
".md": TypeMarkup,
".xml": TypeXml,
".doc": TypeDoc,
".docx": TypeDoc,
".pdf": TypeDoc,
".jpg": TypeImage,
".jpeg": TypeImage,
".png": TypeImage,
".gif": TypeImage,
".html": TypeHtml,
".htm": TypeHtml,
".ogg": TypeAudio,
".mp3": TypeAudio,
".wav": TypeAudio,
".mod": TypeAudio,
".it": TypeAudio,
".xm": TypeAudio,
".mid": TypeAudio,
".vgm": TypeAudio,
".opus": TypeAudio,
".m4a": TypeAudio,
".aac": TypeAudio,
".mp4": TypeVideo,
".mkv": TypeVideo,
".webm": TypeVideo,
}
func buildError(selector string) []byte {

3
fs.go
Unescape Escape View File

@ -270,6 +270,9 @@ func _listDirBase(request *FileSystemRequest, iterFunc func(dirContents *[]byte,
/* Walk through files :D */
for _, file := range files { iterFunc(&dirContents, file) }
/* Append last line */
dirContents = append(dirContents, []byte(LastLine)...)
return dirContents, nil
}

13
gophor.go
Unescape Escape View File

@ -108,9 +108,6 @@ func setupServer() []*GophorListener {
/* Setup the server configuration instance and enter as much as we can right now */
Config = new(ServerConfig)
Config.RootDir = *serverRoot
Config.Description = *serverDescription
Config.AdminEmail = *serverAdmin
Config.Geolocation = *serverGeoloc
Config.PageWidth = *pageWidth
/* Setup Gophor logging system */
@ -175,7 +172,7 @@ func setupServer() []*GophorListener {
}
/* Setup file cache */
Config.FileCache = new(FileCache)
Config.FileSystem = new(FileSystem)
if !*cacheDisabled {
/* Parse suppled cache check frequency time */
@ -185,24 +182,24 @@ func setupServer() []*GophorListener {
}
/* Init file cache */
Config.FileCache.Init(*cacheSize, *cacheFileSizeMax)
Config.FileSystem.Init(*cacheSize, *cacheFileSizeMax)
Config.LogSystem("File caching enabled with: maxcount=%d maxsize=%.3fMB\n", *cacheSize, *cacheFileSizeMax)
/* Before file monitor or any kind of new goroutines started,
* check if we need to cache generated policy files
*/
cachePolicyFiles()
cachePolicyFiles(*serverDescription, *serverAdmin, *serverGeoloc)
/* Start file cache freshness checker */
go startFileMonitor(fileMonitorSleepTime)
Config.LogSystem("File cache freshness monitor started with frequency: %s\n", fileMonitorSleepTime)
} else {
/* File caching disabled, init with zero max size so nothing gets cached */
Config.FileCache.Init(2, 0)
Config.FileSystem.Init(2, 0)
Config.LogSystem("File caching disabled\n")
/* Safe to cache policy files now */
cachePolicyFiles()
cachePolicyFiles(*serverDescription, *serverAdmin, *serverGeoloc)
}
/* Return the created listeners slice :) */

16
policy.go
Unescape Escape View File

@ -4,12 +4,12 @@ import (
"os"
)
func cachePolicyFiles() {
func cachePolicyFiles(description, admin, geoloc string) {
/* See if caps txt exists, if not generate */
_, err := os.Stat("/caps.txt")
if err != nil {
/* We need to generate the caps txt and manually load into cache */
content := generateCapsTxt()
content := generateCapsTxt(description, admin, geoloc)
/* Create new file object from generated file contents */
fileContents := &GeneratedFileContents{ content }
@ -19,7 +19,7 @@ func cachePolicyFiles() {
file.LoadContents()
/* No need to worry about mutexes here, no other goroutines running yet */
Config.FileCache.CacheMap.Put("/caps.txt", file)
Config.FileSystem.CacheMap.Put("/caps.txt", file)
Config.LogSystem("Generated policy file: /caps.txt\n")
}
@ -38,13 +38,13 @@ func cachePolicyFiles() {
file.LoadContents()
/* No need to worry about mutexes here, no other goroutines running yet */
Config.FileCache.CacheMap.Put("/robots.txt", file)
Config.FileSystem.CacheMap.Put("/robots.txt", file)
Config.LogSystem("Generated policy file: /robots.txt\n")
}
}
func generateCapsTxt() []byte {
func generateCapsTxt(description, admin, geoloc string) []byte {
text := "CAPS"+DOSLineEnd
text += DOSLineEnd
text += "# This is an automatically generated"+DOSLineEnd
@ -62,11 +62,11 @@ func generateCapsTxt() []byte {
text += DOSLineEnd
text += "ServerSoftware=Gophor"+DOSLineEnd
text += "ServerSoftwareVersion="+GophorVersion+DOSLineEnd
text += "ServerDescription="+Config.Description+DOSLineEnd
text += "ServerGeolocationString="+Config.Geolocation+DOSLineEnd
text += "ServerDescription="+description+DOSLineEnd
text += "ServerGeolocationString="+geoloc+DOSLineEnd
text += "ServerDefaultEncoding=ascii"+DOSLineEnd
text += DOSLineEnd
text += "ServerAdmin="+Config.AdminEmail+DOSLineEnd
text += "ServerAdmin="+admin+DOSLineEnd
return []byte(text)
}

89
worker.go
Unescape Escape View File

@ -1,7 +1,6 @@
package main
import (
"os"
"path"
"strings"
)
@ -122,90 +121,12 @@ func (worker *Worker) RespondGopher(data []byte) *GophorError {
/* Sanitize supplied path */
requestPath := sanitizePath(dataStr)
/* Handle policy files. TODO: this is so unelegant... */
switch requestPath {
case "/"+CapsTxtStr:
return worker.SendRaw(generateCapsTxt())
case "/"+RobotsTxtStr:
return worker.SendRaw(generateRobotsTxt())
}
/* Open requestPath */
file, err := os.Open(requestPath)
if err != nil {
return &GophorError{ FileOpenErr, err }
}
/* If not empty requestPath, check file type.
* Default type is directory.
*/
fileType := FileTypeDir
if requestPath != "." {
stat, err := file.Stat()
if err != nil {
return &GophorError{ FileStatErr, err }
}
switch {
case stat.Mode() & os.ModeDir != 0:
// do nothing :)
case stat.Mode() & os.ModeType == 0:
fileType = FileTypeRegular
default:
fileType = FileTypeBad
}
}
/* Don't need the file handle anymore */
file.Close()
/* Handle file type. TODO: work on efficiency */
response := make([]byte, 0)
switch fileType {
/* Directory */
case FileTypeDir:
/* First try to serve gopher map */
gophermapPath := path.Join(requestPath, "/"+GophermapFileStr)
fileContents, gophorErr := Config.FileCache.FetchGophermap(&FileSystemRequest{ gophermapPath, worker.Conn.Host })
if gophorErr != nil {
/* Get directory listing instead */
fileContents, gophorErr = listDir(&FileSystemRequest{ requestPath, worker.Conn.Host }, map[string]bool{})
if gophorErr != nil {
return gophorErr
}
/* Add fileContents to response */
response = append(response, fileContents...)
worker.Log("serve dir: %s\n", requestPath)
/* Finish directory listing with LastLine */
response = append(response, []byte(LastLine)...)
} else {
/* Successfully loaded gophermap, add fileContents to response */
response = append(response, fileContents...)
worker.Log("serve gophermap: %s\n", gophermapPath)
}
/* Regular file */
case FileTypeRegular:
/* Read file contents */
fileContents, gophorErr := Config.FileCache.FetchRegular(&FileSystemRequest{ requestPath, worker.Conn.Host })
if gophorErr != nil {
return gophorErr
}
/* Append fileContents to response */
response = append(response, fileContents...)
worker.Log("serve file: %s\n", requestPath)
/* Unsupport file type */
default:
return &GophorError{ FileTypeErr, nil }
}
/* Append lastline */
response = append(response, []byte(LastLine)...)
response, gophorErr := Config.FileSystem.HandleRequest(requestPath, worker.Conn.Host)
if gophorErr != nil {
return gophorErr
}
Config.LogSystem("Served: %s\n", requestPath)
/* Serve response */
return worker.SendRaw(response)

Write Preview
Loading…
Cancel
Save