better conform to URL standards (escape host part too!), do our own escaping, allow user CGI dirs

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

core/config.go

@@ -201,7 +201,7 @@ func ParseConfigAndSetup(tree config.Tree, proto string, defaultPort uint, newLi
 	FileCache = filecache.NewFileCache(int(*cacheSize), *cacheAgeMax)
 
 	// If no restricted paths provided, set to the disabled function. Else, compile and enable
-	if *restrictedPathsList == nil {
+	if len(*restrictedPathsList) == 0 {
 		SystemLog.Info(pathRestrictionsDisabledStr)
 		IsRestrictedPath = isRestrictedPathDisabled
 	} else {
@@ -211,7 +211,7 @@ func ParseConfigAndSetup(tree config.Tree, proto string, defaultPort uint, newLi
 	}
 
 	// If no hidden paths provided, set to the disabled function. Else, compile and enable
-	if *hiddenPathsList == nil {
+	if len(*hiddenPathsList) == 0 {
 		SystemLog.Info(pathHidingDisableStr)
 		IsHiddenPath = isHiddenPathDisabled
 	} else {
@@ -221,7 +221,7 @@ func ParseConfigAndSetup(tree config.Tree, proto string, defaultPort uint, newLi
 	}
 
 	// If no remapped paths provided, set to the disabled function. Else, compile and enable
-	if *remapRequestsList == nil {
+	if len(*remapRequestsList) == 0 {
 		SystemLog.Info(requestRemapDisabledStr)
 		RemapRequest = remapRequestDisabled
 	} else {
@@ -241,16 +241,15 @@ func ParseConfigAndSetup(tree config.Tree, proto string, defaultPort uint, newLi
 		WithinCGIDir = withinCGIDirEnabled
 	}
 
-	// If no user dir supplied, or chroot enabled, set to disabled function.
-	// Else, set user dir and enable
-	if *userSpacesEnabled || *chroot != "" {
-		SystemLog.Info(userSpacesDisabledStr)
-		getRequestPath = getRequestPathUserSpacesDisabled
-	} else {
+	// Set appropriate Path builder function depending
+	// on whether user spaces enabled or disabled
+	if *userSpacesEnabled {
 		SystemLog.Info(userSpacesEnabledStr)
-		getRequestPath = getRequestPathUserSpacesEnabled
-		userSpace = "public_" + proto
-		SystemLog.Infof(userSpacesStr, userSpace)
+		BuildPath = buildPathUserSpacesEnabled
+		SystemLog.Infof(userSpacesStr, "public_"+protocol)
+	} else {
+		SystemLog.Info(userSpacesDisabledStr)
+		BuildPath = buildPathUserSpacesDisabled
 	}
 
 	// Set provided client filesystem handler functions

core/error.go

@@ -9,8 +9,6 @@ var (
 	ConnCloseErr      = errors.NewBaseError(connCloseErrStr)
 	ListenerBeginErr  = errors.NewBaseError(listenerBeginErrStr)
 	ListenerAcceptErr = errors.NewBaseError(listenerAcceptErrStr)
-	InvalidIPErr      = errors.NewBaseError(invalidIPErrStr)
-	InvalidPortErr    = errors.NewBaseError(invalidPortErrStr)
 	MutexUpgradeErr   = errors.NewBaseError(mutexUpgradeErrStr)
 	MutexDowngradeErr = errors.NewBaseError(mutexDowngradeErrStr)
 	FileOpenErr       = errors.NewBaseError(fileOpenErrStr)

core/filesystem.go

@@ -101,6 +101,11 @@ func ScanDirectory(dir *os.File, p *Path, iterator func(os.FileInfo, *Path)) *er
 		// Make new Path object
 		fp := p.JoinPath(name)
 
+		// Skip restricted files
+		if IsRestrictedPath(fp) || IsHiddenPath(fp) {
+			continue
+		}
+
 		// Get stat or continue
 		stat, err := StatFile(fp)
 		if err != nil {
@@ -108,11 +113,6 @@ func ScanDirectory(dir *os.File, p *Path, iterator func(os.FileInfo, *Path)) *er
 			continue
 		}
 
-		// Skip restricted files
-		if IsRestrictedPath(fp) || IsHiddenPath(fp) || WithinCGIDir(fp) {
-			continue
-		}
-
 		// Perform iterator
 		iterator(stat, fp)
 	}

core/path.go

@@ -5,6 +5,9 @@ import (
 	"strings"
 )
 
+// BuildPath is the global Path builder function for a supplied raw (relative) path
+var BuildPath func(string) *Path
+
 // Path safely holds a file path
 type Path struct {
 	root string // root dir
@@ -17,14 +20,38 @@ func NewPath(root, rel string) *Path {
 	return &Path{root, rel, formatSelector(rel)}
 }
 
-// newSanitizedPath returns a new sanitized Path structure based on root and relative path
-func newSanitizedPath(root, rel string) *Path {
-	return NewPath(root, sanitizeRawPath(root, rel))
+// NewSanitizedPathAtRoot returns a new sanitized Path structure based on root and relative path
+func NewSanitizedPathAtRoot(root, rel string) *Path {
+	return NewPath(root, sanitizeRawPath(rel))
+}
+
+// buildPathuserSpacesEnabled will attempt to parse a username, and return a sanitized Path at username's
+// public server dir. Else, returns sanitized Path at server root
+func buildPathUserSpacesEnabled(rawPath string) *Path {
+	if strings.HasPrefix(rawPath, "/~") {
+		// Get username and raw path
+		username, rawPath := SplitByBefore(rawPath[2:], "/")
+
+		// Treat username as a raw path, sanitizing to check for
+		// dir traversals
+		username = sanitizeRawPath(username)
+
+		// Return sanitized path using user home dir as root
+		return NewSanitizedPathAtRoot("/home/"+username+"/public_"+protocol, rawPath)
+	}
+
+	// Return sanitized path at server root
+	return NewSanitizedPathAtRoot(Root, rawPath)
+}
+
+// buildPathUserSpacesDisabled always returns a sanitized Path at server root
+func buildPathUserSpacesDisabled(rawPath string) *Path {
+	return NewSanitizedPathAtRoot(Root, rawPath)
 }
 
 // Remap remaps a Path to a new relative path, keeping previous selector
 func (p *Path) Remap(newRel string) {
-	p.rel = sanitizeRawPath(p.root, newRel)
+	p.rel = sanitizeRawPath(newRel)
 }
 
 // RemapDirect remaps a Path to a new absolute path, keeping previous selector
@@ -97,28 +124,17 @@ func formatSelector(rel string) string {
 }
 
 // sanitizeRawPath takes a root and relative path, and returns a sanitized relative path
-func sanitizeRawPath(root, rel string) string {
+func sanitizeRawPath(raw string) string {
 	// Start by cleaning
-	rel = path.Clean(rel)
-
-	if path.IsAbs(rel) {
-		// Absolute path, try trimming root and leading '/'
-		rel = strings.TrimPrefix(strings.TrimPrefix(rel, root), "/")
-	} else {
-		// Relative path, if back dir traversal give them server root
-		if strings.HasPrefix(rel, "..") {
-			rel = ""
-		}
+	raw = path.Clean(raw)
+
+	// - absolute path --> trim '/'
+	// - back dir ('..') --> return root
+	if path.IsAbs(raw) {
+		raw = raw[1:]
+	} else if strings.HasPrefix(raw, "..") {
+		raw = ""
 	}
 
-	return rel
-}
-
-// sanitizerUserRoot takes a generated user root directory and sanitizes it, returning a bool as to whether it's safe
-func sanitizeUserRoot(root string) (string, bool) {
-	root = path.Clean(root)
-	if !strings.HasPrefix(root, "/home/") && strings.HasSuffix(root, "/"+userSpace) {
-		return "", false
-	}
-	return root, true
+	return raw
 }

core/regex.go

@@ -111,7 +111,7 @@ func compileRequestRemapRegex(remaps []string) []*RequestRemap {
 
 // withinCGIDirEnabled returns whether a Path's absolute value matches within the CGI dir
 func withinCGIDirEnabled(p *Path) bool {
-	return cgiDirRegex.MatchString(p.Absolute())
+	return cgiDirRegex.MatchString(p.Selector())
 }
 
 // withinCGIDirDisabled always returns false, CGI is disabled

core/request.go

@@ -6,6 +6,14 @@ type Request struct {
 	query string
 }
 
+// NewRequest returns a new Request object
+func NewRequest(path *Path, query string) *Request {
+	return &Request{
+		path:  path,
+		query: query,
+	}
+}
+
 // String returns the full parsed request string (mainly for logging)
 func (r *Request) String() string {
 	query := r.query

core/server.go

@@ -41,7 +41,6 @@ var (
 	// File system related globals
 	monitorSleepTime time.Duration
 	fileSizeMax      int64
-	userSpace        string
 
 	// Global listener
 	serverListener *Listener

core/string_constants.go

@@ -70,8 +70,6 @@ const (
 	connCloseErrStr      = "Conn close error"
 	listenerBeginErrStr  = "Listener begin error"
 	listenerAcceptErrStr = "Listener accept error"
-	invalidIPErrStr      = "Invalid IP"
-	invalidPortErrStr    = "Invalid port"
 	mutexUpgradeErrStr   = "Mutex upgrade fail"
 	mutexDowngradeErrStr = "Mutex downgrade fail"
 	fileOpenErrStr       = "File open error"
@@ -80,6 +78,7 @@ const (
 	fileTypeErrStr       = "Unsupported file type"
 	directoryReadErrStr  = "Directory read error"
 	restrictedPathErrStr = "Restricted path"
+	invalidHostErrStr    = "Invalid host"
 	invalidRequestErrStr = "Invalid request"
 	cgiStartErrStr       = "CGI start error"
 	cgiExitCodeErrStr    = "CGI non-zero exit code"

core/url.go

@@ -1,16 +1,11 @@
 package core
 
 import (
-	"net/url"
-	"path"
 	"strings"
 
 	"github.com/grufwub/go-errors"
 )
 
-// getRequestPaths points to either of the getRequestPath____ functions
-var getRequestPath func(string) *Path
-
 // HasAsciiControlBytes returns whether a byte slice contains ASCII control bytes
 func HasAsciiControlBytes(raw string) bool {
 	for i := 0; i < len(raw); i++ {
@@ -56,58 +51,192 @@ func ParseScheme(raw string) (string, string, *errors.Error) {
 	return "", raw, nil
 }
 
-// ParseURLEncodedRequest takes a received string and safely parses a request from this
-func ParseURLEncodedRequest(received string) (*Request, *errors.Error) {
-	// Split into 2 substrings by '?'. URL path and query
-	rawPath, query := SplitBy(received, "?")
+func isHex(b byte) bool {
+	return ('a' <= b && b <= 'f') ||
+		('A' <= b && b <= 'F') ||
+		('0' <= b && b <= '9')
+}
 
-	// Unescape path
-	rawPath, err := url.PathUnescape(rawPath)
-	if err != nil {
-		return nil, errors.WrapError(InvalidRequestErr, err)
+func unHex(b byte) byte {
+	switch {
+	case 'a' <= b || b <= 'f':
+		return b - 'a'
+	case 'A' <= b || b <= 'F':
+		return b - 'A'
+	case '0' <= b || b <= '9':
+		return b - '0'
+	default:
+		return 0
+	}
+}
+
+func shouldEscape(b byte) bool {
+	// All alphanumeric are unreserved
+	if 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z' || '0' <= b && b <= '9' {
+		return false
+	}
+
+	switch b {
+	// Further unreserved
+	case '-', '_', '.', '~':
+		return false
+
+	// All else should be escaped
+	default:
+		return true
 	}
+}
 
-	// Return new request
-	return &Request{getRequestPath(rawPath), query}, nil
+func shouldHostEscape(b byte) bool {
+	switch b {
+	// Allowed host sub-delims +
+	// ':' for port +
+	// '[]' for ipv6 +
+	// '<>' only others we can allow (can't be % encoded)
+	case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', ':', '[', ']', '<', '>', '"':
+		return false
+
+	// Check all-else
+	default:
+		return shouldEscape(b)
+	}
+}
+
+func shouldPathEscape(b byte) bool {
+	switch b {
+	// Reserved character in path
+	case '?':
+		return true
+
+	// Allowed in path
+	case '$', '&', '+', ',', '/', ':', ';', '=', '@':
+		return false
+
+	// Check all-else
+	default:
+		return shouldEscape(b)
+	}
 }
 
-// ParseInternalRequest parses an internal request string based on the current directory
-func ParseInternalRequest(p *Path, line string) *Request {
-	rawPath, query := SplitBy(line, "?")
-	if path.IsAbs(rawPath) {
-		return &Request{getRequestPath(rawPath), query}
+func unescape(raw string, count int) string {
+	var t strings.Builder
+	t.Grow(len(raw) - 2*count)
+	for i := 0; i < len(raw); i++ {
+		switch raw[i] {
+		// Replace % encoded char
+		case '%':
+			t.WriteByte(unHex(raw[i+1])<<4 | unHex(raw[i+2]))
+			i += 2
+
+		// Write as-is
+		default:
+			t.WriteByte(raw[i])
+		}
 	}
-	return &Request{newSanitizedPath(p.Root(), rawPath), query}
+	return t.String()
 }
 
-// getRequestPathUserSpacesEnabled creates a Path object from raw path, converting ~USER to user subdirectory roots, else at server root
-func getRequestPathUserSpacesEnabled(rawPath string) *Path {
-	if userPath := strings.TrimPrefix(rawPath, "/"); strings.HasPrefix(userPath, "~") {
-		// We found a user path! Split into the user part, and remaining path
-		user, remaining := SplitBy(userPath, "/")
+func unescapeHost(raw string) (string, *errors.Error) {
+	// Count all the percent signs
+	count := 0
+	for i := 0; i < len(raw); {
+		switch raw[i] {
+		case '%':
+			// Increase count
+			count++
+
+			// If not a valid % encoded hex value, return with error
+			if i+2 >= len(raw) || !isHex(raw[i+1]) || !isHex(raw[i+2]) {
+				return "", errors.NewError(InvalidRequestErr).Extend("escaping host info " + raw)
+			}
+
+			// In the host component % encoding can only be used
+			// for non-ASCII bytes. And rfc6874 introduces %25 for
+			// escaped percent sign in IPv6 literals
+			if unHex(raw[i+1]) < 8 && raw[i:i+3] != "%25" {
+				return "", errors.NewError(InvalidRequestErr).Extend("escaping host " + raw)
+			}
+
+			// Skip iteration past the
+			// hex we just confirmed
+			i += 3
+		default:
+			// If within ASCII range, and shoud be escaped, return error
+			if raw[i] < 0x80 && shouldHostEscape(raw[i]) {
+				return "", errors.NewError(InvalidRequestErr).Extend("escaping host " + raw)
+			}
 
-		// Empty user, we been duped! Return server root
-		if len(user) <= 1 {
-			return &Path{Root, "", "/"}
+			// Iter
+			i++
 		}
+	}
+
+	// No encoding? return as-is. Else, escape
+	if count == 0 {
+		return raw, nil
+	}
+	return unescape(raw, count), nil
+}
+
+func unescapePath(raw string) (string, *errors.Error) {
+	// Count all the percent signs
+	count := 0
+	for i := 0; i < len(raw); {
+		switch raw[i] {
+		case '%':
+			// Increase count
+			count++
+
+			// If not a valid % encoded hex value, return with error
+			if i+2 >= len(raw) || !isHex(raw[i+1]) || !isHex(raw[i+2]) {
+				return "", errors.NewError(InvalidRequestErr).Extend("escaping path " + raw)
+			}
+
+			// Skip iteration past the
+			// hex we just confirmed
+			i += 3
+		default:
+			// If within ASCII range, and shoud be escaped, return error
+			if raw[i] < 0x80 && shouldPathEscape(raw[i]) {
+				return "", errors.NewError(InvalidRequestErr).Extend("escaping path " + raw)
+			}
 
-		// Get sanitized user root, else return server root
-		root, ok := sanitizeUserRoot(path.Join("/home", user[1:], userSpace))
-		if !ok {
-			return &Path{Root, "", "/"}
+			// Iter
+			i++
 		}
+	}
+
+	// No encoding? return as-is. Else, escape
+	if count == 0 {
+		return raw, nil
+	}
+	return unescape(raw, count), nil
+}
 
-		// Build new Path
-		rel := sanitizeRawPath(root, remaining)
-		sel := "/~" + user[1:] + formatSelector(rel)
-		return &Path{root, rel, sel}
+// ParseEncodedHost parses encoded host info, safely returning unescape host and port
+func ParseEncodedHost(raw string) (string, string, *errors.Error) {
+	// Unescape the host info
+	raw, err := unescapeHost(raw)
+	if err != nil {
+		return "", "", err
 	}
 
-	// Return regular server root + rawPath
-	return newSanitizedPath(Root, rawPath)
+	// Split by last ':' and return
+	host, port := SplitByLast(raw, ":")
+	return host, port, nil
 }
 
-// getRequestPathUserSpacesDisabled creates a Path object from raw path, always at server root
-func getRequestPathUserSpacesDisabled(rawPath string) *Path {
-	return newSanitizedPath(Root, rawPath)
+// ParseEncodedURI parses encoded URI, safely returning unescaped path and still-escaped query
+func ParseEncodedURI(received string) (string, string, *errors.Error) {
+	// Split into path and query
+	rawPath, query := SplitBy(received, "?")
+
+	// Unescape path, query is up-to CGI scripts
+	rawPath, err := unescapePath(rawPath)
+	if err != nil {
+		return "", "", errors.WrapError(InvalidRequestErr, err)
+	}
+
+	// Return the raw path and query
+	return rawPath, query, nil
 }

core/util.go

@@ -21,7 +21,7 @@ func SplitBy(input, delim string) (string, string) {
 	return input[:index], input[index+len(delim):]
 }
 
-// SplitBy takes an input string and a delimiter, returning resulting two string from split with the delim at end of 2nd
+// SplitBy takes an input string and a delimiter, returning resulting two string from split with the delim at beginning of 2nd
 func SplitByBefore(input, delim string) (string, string) {
 	index := strings.Index(input, delim)
 	if index == -1 {

gemini/server.go

@@ -43,47 +43,43 @@ func serve(client *core.Client) {
 
 	// Infer no schema as 'gemini', else check we
 	// were explicitly provided 'gemini'
-	if len(scheme) > 0 {
+	if scheme != "" {
 		if scheme != "gemini" {
 			client.LogError(clientInvalidRequestStr)
-			handleError(client, errors.NewError(invalidProtocolErr))
+			handleError(client, errors.NewError(invalidProtocolErr).Extend(scheme))
 			return
 		}
 	}
 
-	// Split by first '/' (with prefix '//' trimmed) to get hostPort string, and path
-	hostPort, path := core.SplitByBefore(strings.TrimPrefix(path, "//"), "/")
+	// Split by first '/' (with prefix '//' trimmed) to get host info and path strings
+	host, path := core.SplitByBefore(strings.TrimPrefix(path, "//"), "/")
 
-	// If we have empty hostPort, bad request!
-	if len(hostPort) == 0 {
+	// Parse the URL encoded host info
+	host, port, err := core.ParseEncodedHost(host)
+	if err != nil {
 		client.LogError(clientInvalidRequestStr)
-		handleError(client, errors.NewError(core.InvalidRequestErr))
+		handleError(client, err)
 		return
 	}
 
-	// Ensure supplied host matches our own, and supplied
-	// port is either empty, or again matches ours
-	if host, port := core.SplitByLast(hostPort, ":"); host != core.Hostname || (port != "" && port != core.Port) {
+	// Check the host and port are our own (empty port is allowed)
+	if host != core.Hostname || (port != "" && port != core.Port) {
 		client.LogError(clientInvalidRequestStr)
-		handleError(client, errors.NewError(invalidHostPortErr))
-		return
-	}
-
-	// Empty path redirect to root
-	if len(path) == 0 {
-		client.LogInfo(clientRedirectStr, path, "/")
-		client.Conn().Write(buildRedirect("gemini://" + core.Hostname + ":" + core.Port + "/"))
+		handleError(client, errors.NewError(invalidHostPortErr).Extend(host+" "+port))
 		return
 	}
 
-	// Parse new request
-	request, err := core.ParseURLEncodedRequest(path)
+	// Parse the encoded URI into path and query components
+	path, query, err := core.ParseEncodedURI(path)
 	if err != nil {
-		client.LogError(clientRequestParseFailStr)
+		client.LogError(clientInvalidRequestStr)
 		handleError(client, err)
 		return
 	}
 
+	// Build new Request from raw path and query
+	request := core.NewRequest(core.BuildPath(path), query)
+
 	// Handle the request!
 	err = core.HandleClient(client, request)
 

gopher/gophermap.go

@@ -58,16 +58,21 @@ func readGophermap(file *os.File, p *core.Path) ([]gophermapSection, *errors.Err
 				return true
 
 			case typeSubGophermap:
-				// Parse new Path and parameters
-				request := core.ParseInternalRequest(p, line[1:])
+				// Parse encoded URI
+				path, query, returnErr := core.ParseEncodedURI(line[1:])
 				if returnErr != nil {
 					return false
-				} else if request.Path().Relative() == "" || request.Path().Relative() == p.Relative() {
+				}
+
+				// Build new request. If empty relative path, or relative
+				// equal to current gophermap (recurse!!!) we return error
+				request := core.NewRequest(core.BuildPath(path), query)
+				if request.Path().Relative() == "" || request.Path().Relative() == p.Relative() {
 					returnErr = errors.NewError(InvalidGophermapErr)
 					return false
 				}
 
-				// Open FD
+				// Open sub gophermap
 				var subFile *os.File
 				subFile, returnErr = core.OpenFile(request.Path())
 				if returnErr != nil {

gopher/server.go

@@ -24,7 +24,11 @@ func serve(client *core.Client) {
 		handleError(client, err)
 		return
 	}
-	raw := string(received)
+
+	// Split up to first tab in case we've been
+	// given index search query (which we use to set CGI env),
+	// or extra Gopher+ information (which we don't care about)
+	raw, extra := core.SplitBy(string(received), "\t")
 
 	// Ensure we've received a valid URL string
 	if core.HasAsciiControlBytes(raw) {
@@ -33,31 +37,24 @@ func serve(client *core.Client) {
 		return
 	}
 
-	// Trim leading '/'
-	raw = strings.TrimPrefix(raw, "/")
+	// Parse the encoded URI into path and query components
+	path, query, err := core.ParseEncodedURI(raw)
+	if err != nil {
+		client.LogError(clientRequestParseFailStr)
+		handleError(client, err)
+		return
+	}
 
 	// If prefixed by 'URL:' send a redirect
-	if strings.HasPrefix(raw, "URL:") {
-		raw = raw[4:]
+	if strings.HasPrefix(path, "/URL:") {
+		raw = raw[5:]
 		client.Conn().Write(generateHTMLRedirect(raw))
 		client.LogInfo(clientRedirectFmtStr, raw)
 		return
 	}
 
-	// Split up to first tab in case we've been
-	// given index search query (which we use to set CGI env),
-	// or extra Gopher+ information (which we don't handle)
-	raw, extra := core.SplitBy(raw, "\t")
-
-	// Parse new request
-	request, err := core.ParseURLEncodedRequest(raw)
-	if err != nil {
-		client.LogError(clientRequestParseFailStr)
-		handleError(client, err)
-		return
-	}
-
-	// Add extra to request query
+	// Create new request and add the extra query part
+	request := core.NewRequest(core.BuildPath(path), query)
 	request.AddToQuery(extra)
 
 	// Handle the request!